hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE] mds_mvs

Browse Source 
Co-authored-by: xuhuleon <xuhuleon@qq.com>
Co-authored-by: godyangfight <godyangfight@gmail.com>
Co-authored-by: JiahuaChen <garfieldjia@qq.com>
This commit is contained in:
hiddenbomb 2024-06-25 07:51:39 +00:00 committed by ob-robot
parent 8e6c6a9c4d
commit c981050104
288 changed files with 16208 additions and 6879 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
deps/oblib/src/common/meta_programming/ob_meta_swap.h vendored Normal file
View File

@ -0,0 +1,57 @@
/**
* Copyright (c) 2023 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef SRC_COMMON_META_PROGRAMMING_OB_META_SWAP_H
#define SRC_COMMON_META_PROGRAMMING_OB_META_SWAP_H
#include "ob_type_traits.h"
namespace oceanbase
{
namespace common
{
namespace meta
{
// try use member swap() method first
template <typename T, typename Member, ENABLE_IF_SWAPABLE(Member)>
void swap(T &lhs, T &rhs, Member &member) noexcept
{
if (OB_LIKELY((char *)&member >= (char *)&lhs && (char *)&member < (char *)&lhs + sizeof(lhs))) {
member.swap(*(Member *)((char *)&rhs + ((char *)&member - (char *)&lhs)));
} else {
ob_abort();
}
}
// otherwise decay to standard swap, that is : temp = a; a = b; b = temp;
template <typename T, typename Member, ENABLE_IF_NOT_SWAPABLE(Member)>
void swap(T &lhs, T &rhs, Member &member) noexcept
{
if (OB_LIKELY((char *)&member >= (char *)&lhs && (char *)&member < (char *)&lhs + sizeof(lhs))) {
std::swap(member, *(Member *)((char *)&rhs + ((char *)&member - (char *)&lhs)));
} else {
ob_abort();
}
}
template <typename T, typename Head, typename ...Others>
void swap(T &lhs, T &rhs, Head &head_member, Others &...other_members) noexcept
{
swap(lhs, rhs, head_member);
swap(lhs, rhs, other_members...);
}
}
}
}
#endif

30
deps/oblib/src/common/meta_programming/ob_type_traits.h vendored
View File

@ -132,8 +132,12 @@ REGISTER_FUNCTION_TRAIT(serialize)
REGISTER_FUNCTION_TRAIT(deserialize)
REGISTER_FUNCTION_TRAIT(get_serialize_size)
REGISTER_FUNCTION_TRAIT(compare)
REGISTER_FUNCTION_TRAIT(swap)
REGISTER_FUNCTION_TRAIT(check_can_replay_commit)
REGISTER_FUNCTION_TRAIT(on_commit_for_old_mds)
REGISTER_FUNCTION_TRAIT(mds_serialize)
REGISTER_FUNCTION_TRAIT(mds_deserialize)
REGISTER_FUNCTION_TRAIT(mds_get_serialize_size)
REGISTER_FUNCTION_TRAIT(on_set)
REGISTER_FUNCTION_TRAIT(on_redo)
@ -186,6 +190,18 @@ typename std::enable_if<OB_TRAIT_DEEP_SERIALIZEABLE(CLASS), bool>::type = true
#define ENABLE_IF_NOT_DEEP_SERIALIZEABLE(CLASS) \
typename std::enable_if<!OB_TRAIT_DEEP_SERIALIZEABLE(CLASS), bool>::type = true
// define MDS serialize trait and enable_if macro
#define OB_TRAIT_MDS_SERIALIZEABLE(CLASS) \
(::oceanbase::common::meta::has_mds_serialize<DECAY(CLASS),\
int(char*, const int64_t, int64_t &)>::value &&\
::oceanbase::common::meta::has_mds_deserialize<DECAY(CLASS),\
int(const char*, const int64_t, int64_t &)>::value &&\
::oceanbase::common::meta::has_mds_get_serialize_size<DECAY(CLASS), int64_t()>::value)
#define ENABLE_IF_MDS_SERIALIZEABLE(CLASS) \
typename std::enable_if<OB_TRAIT_MDS_SERIALIZEABLE(CLASS), bool>::type = true
#define ENABLE_IF_NOT_MDS_SERIALIZEABLE(CLASS) \
typename std::enable_if<!OB_TRAIT_MDS_SERIALIZEABLE(CLASS), bool>::type = true
// define assign trait and enable_if macro
#define OB_TRAIT_HAS_ASSIGN(CLASS) \
::oceanbase::common::meta::has_assign<DECAY(CLASS), int(const CLASS &)>::value
@ -226,16 +242,24 @@ typename std::enable_if<!::oceanbase::common::meta::has_##FUNCTION<DECAY(CLASS),
#define OB_TRAIT_HAS_EQUAL_OPERATOR(CLASS) \
::oceanbase::common::meta::has_equal_operator<DECAY(CLASS)>::value
#define OB_TRAIT_IS_ORIGIN_COMPAREABLE(CLASS) \
OB_TRAIT_HAS_LESS_OPERATOR(CLASS) && OB_TRAIT_HAS_EQUAL_OPERATOR(CLASS)
(OB_TRAIT_HAS_LESS_OPERATOR(CLASS) && OB_TRAIT_HAS_EQUAL_OPERATOR(CLASS))
#define OB_TRAIT_IS_METHOD_COMPAREABLE(CLASS) \
::oceanbase::common::meta::has_compare<DECAY(CLASS), int(const CLASS &)>::value
#define OB_TRAIT_IS_COMPAREABLE(CLASS) \
OB_TRAIT_IS_ORIGIN_COMPAREABLE(CLASS) || OB_TRAIT_IS_METHOD_COMPAREABLE(CLASS)
(OB_TRAIT_IS_ORIGIN_COMPAREABLE(CLASS) || OB_TRAIT_IS_METHOD_COMPAREABLE(CLASS))
#define ENABLE_IF_COMPAREABLE(CLASS) \
typename std::enable_if<OB_TRAIT_IS_COMPAREABLE(CLASS), bool>::type = true
#define ENABLE_IF_NOT_COMPAREABLE(CLASS) \
typename std::enable_if<!OB_TRAIT_IS_COMPAREABLE(CLASS), bool>::type = true
// define swap trait and enable_if macro
#define OB_TRAIT_IS_SWAPABLE(CLASS) \
::oceanbase::common::meta::has_swap<DECAY(CLASS), void(CLASS &)>::value
#define ENABLE_IF_SWAPABLE(CLASS) \
typename std::enable_if<OB_TRAIT_IS_SWAPABLE(CLASS), bool>::type = true
#define ENABLE_IF_NOT_SWAPABLE(CLASS) \
typename std::enable_if<!OB_TRAIT_IS_SWAPABLE(CLASS), bool>::type = true
// define copy and move trait and enable_if macro
#define OB_TRAIT_IS_MOVEABLE(CLASS) \
std::is_move_assignable<DECAY(T)>::value || std::is_move_constructible<DECAY(T)>::value
@ -244,7 +268,7 @@ OB_TRAIT_HAS_ASSIGN(T) ||\
std::is_copy_assignable<DECAY(T)>::value ||\
std::is_copy_constructible<DECAY(T)>::value
#define OB_TRAIT_IS_MOVE_OR_COPIABLE(CLASS) \
OB_TRAIT_IS_MOVEABLE(CLASS) || OB_TRAIT_IS_COPIABLE(CLASS)
(OB_TRAIT_IS_MOVEABLE(CLASS) || OB_TRAIT_IS_COPIABLE(CLASS))
#define ENABLE_IF_MOVEABLE(CLASS, FUNCTION, DECLEARATION) \
typename std::enable_if<OB_TRAIT_IS_MOVEABLE(CLASS), bool>::type = true
#define ENABLE_IF_NOT_MOVEABLE(CLASS, FUNCTION, DECLEARATION) \

4
deps/oblib/src/common/row/ob_row_iterator.h vendored
View File

@ -57,7 +57,7 @@ public:
// Iterate row interface for sql static typing engine.
virtual int get_next_row()
{
int ret = common::OB_NOT_IMPLEMENT;;
int ret = common::OB_NOT_IMPLEMENT;
COMMON_LOG(WARN, "interface not implement", K(ret));
return ret;
}
@ -66,7 +66,7 @@ public:
virtual int get_next_rows(int64_t &count, int64_t capacity)
{
UNUSEDx(count, capacity);
int ret = common::OB_NOT_IMPLEMENT;;
int ret = common::OB_NOT_IMPLEMENT;
COMMON_LOG(WARN, "interface not implement", K(ret));
return ret;
}

24
deps/oblib/src/lib/mysqlclient/ob_mysql_result.h vendored
View File

@ -719,6 +719,30 @@
} \
}
#define EXTRACT_VARCHAR_FIELD_MYSQL_SKIP_RET_WITH_COLUMN_INFO(result, column_name, field, column_is_null, column_not_exist) \
if (OB_SUCC(ret)) \
{ \
column_is_null = false; \
column_not_exist = false; \
if (OB_SUCCESS != (ret = (result).get_varchar(column_name, field))) \
{ \
if (OB_ERR_NULL_VALUE == ret) \
{ \
column_is_null = true; \
ret = OB_SUCCESS; \
} \
else if (OB_ERR_COLUMN_NOT_FOUND == ret) \
{ \
column_not_exist = true; \
ret = OB_SUCCESS; \
} \
else \
{ \
SQL_LOG(WARN, "get varchar failed", KR(ret)); \
} \
} \
}
// Macro with default value
// 1. skip_null_error: indicates whether to ignore NULL values
// 2. skip_column_error: indicates whether to ignore column errors, and pass in ObSchemaService::g_ignore_column_retrieve_error_

1
deps/oblib/src/lib/roaringbitmap/ob_rb_utils.cpp vendored
View File

@ -14,6 +14,7 @@
#define USING_LOG_PREFIX LIB
#include "lib/ob_errno.h"
#include "lib/utility/ob_sort.h"
#include "lib/utility/ob_macro_utils.h"
#include "lib/utility/ob_fast_convert.h"
#include "lib/roaringbitmap/ob_rb_utils.h"

2
mittest/mtlenv/CMakeLists.txt
View File

@ -6,4 +6,4 @@ storage_unittest(test_buffer_ctx_node test_buffer_ctx_node.cpp)
ob_unittest(test_htable_lock test_htable_lock.cpp)
add_subdirectory(storage)
add_subdirectory(tablelock)
add_subdirectory(tablelock)

7
mittest/mtlenv/mock_tenant_module_env.h
View File

@ -48,6 +48,7 @@
#include "sql/ob_sql.h"
#include "storage/blocksstable/ob_log_file_spec.h"
#include "storage/blocksstable/ob_decode_resource_pool.h"
#include "storage/compaction/ob_compaction_tablet_diagnose.h"
#include "storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.h"
#include "storage/slog_ckpt/ob_server_checkpoint_slog_handler.h"
#include "storage/meta_mem/ob_tenant_meta_mem_mgr.h"
@ -63,6 +64,7 @@
#include "storage/tx_storage/ob_tenant_freezer.h"
#include "storage/tx_storage/ob_access_service.h"
#include "storage/lob/ob_lob_manager.h"
#include "storage/tablet/ob_mds_schema_helper.h"
#include "storage/tx/ob_ts_mgr.h"
#include "storage/tx/ob_xa_service.h"
#include "storage/tx/ob_trans_service.h"
@ -656,10 +658,12 @@ int MockTenantModuleEnv::init_before_start_mtl()
STORAGE_LOG(WARN, "fail to init env", K(ret));
} else if (OB_FAIL(oceanbase::palf::election::GLOBAL_INIT_ELECTION_MODULE())) {
STORAGE_LOG(WARN, "fail to init env", K(ret));
} else if (OB_SUCCESS != (ret = bandwidth_throttle_.init(1024 *1024 * 60))) {
} else if (OB_SUCCESS != (ret = bandwidth_throttle_.init(1024 * 1024 * 60))) {
STORAGE_LOG(ERROR, "failed to init bandwidth_throttle_", K(ret));
} else if (OB_FAIL(TG_START(lib::TGDefIDs::ServerGTimer))) {
STORAGE_LOG(ERROR, "init timer fail", KR(ret));
} else if (OB_FAIL(ObMdsSchemaHelper::get_instance().init())) {
STORAGE_LOG(ERROR, "fail to init mds schema helper", K(ret));
} else {
obrpc::ObRpcNetHandler::CLUSTER_ID = 1;
oceanbase::palf::election::INIT_TS = 1;
@ -710,6 +714,7 @@ int MockTenantModuleEnv::init()
MTL_BIND2(mtl_new_default, ObTenantCheckpointSlogHandler::mtl_init, nullptr, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, coordinator::ObLeaderCoordinator::mtl_init, coordinator::ObLeaderCoordinator::mtl_start, coordinator::ObLeaderCoordinator::mtl_stop, coordinator::ObLeaderCoordinator::mtl_wait, mtl_destroy_default);
MTL_BIND2(mtl_new_default, coordinator::ObFailureDetector::mtl_init, coordinator::ObFailureDetector::mtl_start, coordinator::ObFailureDetector::mtl_stop, coordinator::ObFailureDetector::mtl_wait, mtl_destroy_default);
MTL_BIND2(mtl_new_default, compaction::ObDiagnoseTabletMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND2(ObLobManager::mtl_new, mtl_init_default, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, share::detector::ObDeadLockDetectorMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND2(mtl_new_default, storage::ObTenantTabletStatMgr::mtl_init, nullptr, mtl_stop_default, mtl_wait_default, mtl_destroy_default)

7
mittest/mtlenv/storage/CMakeLists.txt
View File

@ -3,7 +3,6 @@ storage_unittest(test_ls_service test_ls_service.cpp)
storage_dml_unittest(test_write_tablet_slog test_write_tablet_slog.cpp)
storage_dml_unittest(test_table_scan_pure_data_table)
storage_dml_unittest(test_tenant_meta_mem_mgr test_tenant_meta_mem_mgr.cpp)
#storage_dml_unittest(test_tablet_status test_tablet_status.cpp) TODO yq: transfer fix later
storage_dml_unittest(test_tablet_status_cache test_tablet_status_cache.cpp)
storage_dml_unittest(test_tablet_member_load_and_free test_tablet_member_load_and_free.cpp)
storage_dml_unittest(test_ls_migration_param test_ls_migration_param.cpp)
@ -19,10 +18,14 @@ storage_dml_unittest(test_index_sstable_multi_estimator)
storage_dml_unittest(test_multi_version_sstable_single_get)
storage_dml_unittest(test_multi_version_sstable_merge)
storage_dml_unittest(test_co_merge)
storage_dml_unittest(test_ddl_create_tablet test_ddl_create_tablet.cpp)
storage_dml_unittest(test_medium_info_iterator test_medium_info_iterator.cpp)
storage_dml_unittest(test_medium_info_reader test_medium_info_reader.cpp)
storage_dml_unittest(test_ddl_create_tablet test_ddl_create_tablet.cpp)
storage_dml_unittest(test_migration_tablet_param test_migration_tablet_param.cpp)
storage_dml_unittest(test_tablet_mds_data test_tablet_mds_data.cpp)
storage_dml_unittest(test_mds_data_read_write test_mds_data_read_write.cpp)
storage_dml_unittest(test_mds_table_scan test_mds_table_scan.cpp)
storage_dml_unittest(test_mds_compat test_mds_compat.cpp)
storage_unittest(test_physical_copy_task test_physical_copy_task.cpp)
storage_unittest(test_shared_block_reader_writer)
storage_dml_unittest(test_meta_snapshot test_meta_snapshot.cpp)

247
mittest/mtlenv/storage/medium_info_common.h Normal file
View File

@ -0,0 +1,247 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_UNITTEST_MEDIUM_INFO_COMMON
#define OCEANBASE_UNITTEST_MEDIUM_INFO_COMMON
#include <gtest/gtest.h>
#define private public
#define protected public
#include "lib/oblog/ob_log.h"
#include "lib/allocator/page_arena.h"
#include "share/ob_ls_id.h"
#include "common/ob_tablet_id.h"
#include "mtlenv/mock_tenant_module_env.h"
#include "mtlenv/storage/medium_info_helper.h"
#include "share/rc/ob_tenant_base.h"
#include "unittest/storage/test_tablet_helper.h"
#include "unittest/storage/test_dml_common.h"
#include "unittest/storage/init_basic_struct.h"
#include "unittest/storage/schema_utils.h"
#include "storage/multi_data_source/mds_table_handler.h"
#include "storage/multi_data_source/runtime_utility/mds_factory.h"
using namespace oceanbase::common;
using namespace oceanbase::share;
using namespace oceanbase::unittest;
namespace oceanbase
{
namespace storage
{
class MediumInfoCommon : public ::testing::Test
{
public:
MediumInfoCommon() = default;
virtual ~MediumInfoCommon() = default;
public:
virtual void SetUp() override;
virtual void TearDown() override;
static void SetUpTestCase();
static void TearDownTestCase();
public:
static int create_ls(const uint64_t tenant_id, const share::ObLSID &ls_id, ObLSHandle &ls_handle);
static int remove_ls(const share::ObLSID &ls_id);
int create_tablet(const common::ObTabletID &tablet_id, ObTabletHandle &tablet_handle);
int insert_medium_info(const int64_t trans_id, const compaction::ObMediumCompactionInfoKey &key);
static int get_tablet(const common::ObTabletID &tablet_id, ObTabletHandle &tablet_handle);
int wait_for_mds_table_flush(const common::ObTabletID &tablet_id);
int wait_for_all_mds_nodes_released(const common::ObTabletID &tablet_id);
public:
static constexpr uint64_t TENANT_ID = 1001;
static const share::ObLSID LS_ID;
mds::MdsTableHandle mds_table_;
common::ObArenaAllocator allocator_;
};
const share::ObLSID MediumInfoCommon::LS_ID(1234);
void MediumInfoCommon::SetUp()
{
}
void MediumInfoCommon::TearDown()
{
}
void MediumInfoCommon::SetUpTestCase()
{
int ret = OB_SUCCESS;
ret = MockTenantModuleEnv::get_instance().init();
ASSERT_EQ(OB_SUCCESS, ret);
ObServerCheckpointSlogHandler::get_instance().is_started_ = true;
// create ls
ObLSHandle ls_handle;
ret = create_ls(TENANT_ID, LS_ID, ls_handle);
ASSERT_EQ(OB_SUCCESS, ret);
}
void MediumInfoCommon::TearDownTestCase()
{
int ret = OB_SUCCESS;
// remove ls
ret = remove_ls(LS_ID);
ASSERT_EQ(OB_SUCCESS, ret);
MockTenantModuleEnv::get_instance().destroy();
}
int MediumInfoCommon::create_ls(const uint64_t tenant_id, const share::ObLSID &ls_id, ObLSHandle &ls_handle)
{
int ret = OB_SUCCESS;
ret = TestDmlCommon::create_ls(tenant_id, ls_id, ls_handle);
return ret;
}
int MediumInfoCommon::remove_ls(const share::ObLSID &ls_id)
{
int ret = OB_SUCCESS;
ret = MTL(ObLSService*)->remove_ls(ls_id);
return ret;
}
int MediumInfoCommon::create_tablet(const common::ObTabletID &tablet_id, ObTabletHandle &tablet_handle)
{
int ret = OB_SUCCESS;
const uint64_t table_id = 1234567;
share::schema::ObTableSchema table_schema;
ObLSHandle ls_handle;
ObLS *ls = nullptr;
mds_table_.reset();
if (OB_FAIL(MTL(ObLSService*)->get_ls(LS_ID, ls_handle, ObLSGetMod::STORAGE_MOD))) {
LOG_WARN("failed to get ls", K(ret));
} else if (OB_ISNULL(ls = ls_handle.get_ls())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls is null", K(ret), KP(ls));
} else if (OB_FAIL(build_test_schema(table_schema, table_id))) {
LOG_WARN("failed to build table schema");
} else if (OB_FAIL(TestTabletHelper::create_tablet(ls_handle, tablet_id, table_schema, allocator_))) {
LOG_WARN("failed to create tablet", K(ret));
} else if (OB_FAIL(ls->get_tablet(tablet_id, tablet_handle))) {
LOG_WARN("failed to get tablet", K(ret));
} else if (OB_FAIL(tablet_handle.get_obj()->inner_get_mds_table(mds_table_, true/*not_exist_create*/))) {
LOG_WARN("failed to get mds table", K(ret));
}
return ret;
}
int MediumInfoCommon::insert_medium_info(const int64_t trans_id, const compaction::ObMediumCompactionInfoKey &key)
{
int ret = OB_SUCCESS;
mds::MdsCtx ctx{mds::MdsWriter{transaction::ObTransID{trans_id}}};
compaction::ObMediumCompactionInfo info;
if (OB_FAIL(MediumInfoHelper::build_medium_compaction_info(allocator_, info, trans_id))) {
LOG_WARN("fail to build medium info", K(ret), K(trans_id));
} else if (OB_FAIL(mds_table_.set(key, info, ctx))) {
LOG_WARN("fail to write data to mds table", K(ret), K(key), K(info));
} else {
const share::SCN &scn = mock_scn(trans_id);
ctx.single_log_commit(scn, scn);
}
return ret;
}
int MediumInfoCommon::get_tablet(const common::ObTabletID &tablet_id, ObTabletHandle &tablet_handle)
{
int ret = OB_SUCCESS;
ObLSHandle ls_handle;
ObLS *ls = nullptr;
if (OB_FAIL(MTL(ObLSService*)->get_ls(LS_ID, ls_handle, ObLSGetMod::STORAGE_MOD))) {
LOG_WARN("failed to get ls", K(ret));
} else if (OB_ISNULL(ls = ls_handle.get_ls())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls is null", K(ret), KP(ls));
} else if (OB_FAIL(ls->get_tablet_svr()->direct_get_tablet(tablet_id, tablet_handle))) {
LOG_WARN("failed to get tablet", K(ret), KP(ls));
}
return ret;
}
int MediumInfoCommon::wait_for_mds_table_flush(const common::ObTabletID &tablet_id)
{
int ret = OB_SUCCESS;
int times = 0;
share::SCN rec_scn = share::SCN::min_scn();
do
{
ret = mds_table_.get_rec_scn(rec_scn);
EXPECT_EQ(OB_SUCCESS, ret);
// sleep
::ob_usleep(100_ms);
++times;
} while (OB_SUCCESS == ret && !rec_scn.is_max() && times < 20);
EXPECT_TRUE(rec_scn.is_max());
// check mds sstable
ObTabletHandle tablet_handle;
ObTablet *tablet = nullptr;
ret = MediumInfoCommon::get_tablet(tablet_id, tablet_handle);
EXPECT_EQ(OB_SUCCESS, ret);
tablet = tablet_handle.get_obj();
EXPECT_NE(nullptr, tablet);
ObTabletMemberWrapper<ObTabletTableStore> table_store_wrapper;
ret = tablet->fetch_table_store(table_store_wrapper);
EXPECT_EQ(OB_SUCCESS, ret);
const ObTabletTableStore *table_store = table_store_wrapper.get_member();
EXPECT_EQ(1, table_store->mds_sstables_.count());
if (::testing::Test::HasFailure()) {
ret = OB_TIMEOUT;
}
return ret;
}
int MediumInfoCommon::wait_for_all_mds_nodes_released(const common::ObTabletID &tablet_id)
{
int ret = OB_SUCCESS;
ObTabletHandle tablet_handle;
ObTablet *tablet = nullptr;
int times = 0;
int64_t node_cnt = INT64_MAX;
do {
ret = mds_table_.get_node_cnt(node_cnt);
EXPECT_EQ(OB_SUCCESS, ret);
// sleep
::ob_usleep(100_ms);
++times;
} while (OB_SUCCESS == ret && node_cnt != 0 && times < 60);
EXPECT_EQ(0, node_cnt);
if (::testing::Test::HasFailure()) {
ret = OB_TIMEOUT;
}
return ret;
}
} // namespace storage
} // namespace oceanbase
#endif // OCEANBASE_UNITTEST_MEDIUM_INFO_COMMON

40
mittest/mtlenv/storage/test_ls_migration_param.cpp
View File

@ -38,6 +38,11 @@ using namespace oceanbase::share::schema;
namespace oceanbase
{
int ObClusterVersion::get_tenant_data_version(const uint64_t tenant_id, uint64_t &data_version)
{
data_version = DATA_VERSION_4_3_2_0;
return OB_SUCCESS;
}
namespace storage
{
class TestLSMigrationParam : public ::testing::Test
@ -251,6 +256,21 @@ TEST_F(TestLSMigrationParam, test_migrate_tablet_param)
scn, 2022, create_tablet_schema, true/*need_create_empty_major_sstable*/, ls_handle.get_ls()->get_freezer());
ASSERT_EQ(common::OB_SUCCESS, ret);
share::SCN create_commit_scn;
create_commit_scn = share::SCN::plus(share::SCN::min_scn(), 50);
// write data to mds table no.1 row
{
ObTabletCreateDeleteMdsUserData user_data;
user_data.tablet_status_ = ObTabletStatus::NORMAL;
user_data.data_type_ = ObTabletMdsUserDataType::CREATE_TABLET;
mds::MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(123)));
ret = src_handle.get_obj()->set_tablet_status(user_data, ctx);
ASSERT_EQ(OB_SUCCESS, ret);
ctx.single_log_commit(create_commit_scn, create_commit_scn);
}
ObMigrationTabletParam tablet_param;
ret = src_handle.get_obj()->build_migration_tablet_param(tablet_param);
ASSERT_EQ(OB_SUCCESS, ret);
@ -264,8 +284,9 @@ TEST_F(TestLSMigrationParam, test_migrate_tablet_param)
ret = t3m->create_tmp_tablet(WashTabletPriority::WTP_HIGH, dst_key, allocator_, ls_handle, dst_handle);
ASSERT_EQ(common::OB_SUCCESS, ret);
ret = dst_handle.get_obj()->init_with_migrate_param(allocator_, tablet_param, false, ls_handle.get_ls()->get_freezer());
ASSERT_EQ(common::OB_SUCCESS, ret);
const bool is_transfer = false;
ret = dst_handle.get_obj()->init_with_migrate_param(allocator_, tablet_param, false, ls_handle.get_ls()->get_freezer(), is_transfer);
ASSERT_EQ(OB_SUCCESS, ret);
const ObTabletMeta &src_meta = src_handle.get_obj()->get_tablet_meta();
const ObTabletMeta &dst_meta = dst_handle.get_obj()->get_tablet_meta();
@ -316,6 +337,21 @@ TEST_F(TestLSMigrationParam, test_migration_param_compat)
scn, 2022, create_tablet_schema, true/*need_create_empty_major_sstable*/, ls_handle.get_ls()->get_freezer());
ASSERT_EQ(common::OB_SUCCESS, ret);
share::SCN create_commit_scn;
create_commit_scn = share::SCN::plus(share::SCN::min_scn(), 50);
// write data to mds table no.1 row
{
ObTabletCreateDeleteMdsUserData user_data;
user_data.tablet_status_ = ObTabletStatus::NORMAL;
user_data.data_type_ = ObTabletMdsUserDataType::CREATE_TABLET;
mds::MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(123)));
ret = src_handle.get_obj()->set_tablet_status(user_data, ctx);
ASSERT_EQ(OB_SUCCESS, ret);
ctx.single_log_commit(create_commit_scn, create_commit_scn);
}
ObMigrationTabletParam tablet_param;
ret = src_handle.get_obj()->build_migration_tablet_param(tablet_param);
ASSERT_EQ(OB_SUCCESS, ret);

20
mittest/mtlenv/storage/test_ls_tablet_info_writer_and_reader.cpp
View File

@ -38,6 +38,11 @@ using namespace oceanbase::share::schema;
namespace oceanbase
{
int ObClusterVersion::get_tenant_data_version(const uint64_t tenant_id, uint64_t &data_version)
{
data_version = DATA_VERSION_4_3_2_0;
return OB_SUCCESS;
}
namespace storage
{
class TestLSTabletInfoWR : public ::testing::Test
@ -168,6 +173,21 @@ void TestLSTabletInfoWR::fill_tablet_meta()
scn, 2022, create_tablet_schema, true/*need_create_empty_major_sstable*/, ls_handle.get_ls()->get_freezer());
ASSERT_EQ(common::OB_SUCCESS, ret);
share::SCN create_commit_scn;
create_commit_scn = share::SCN::plus(share::SCN::min_scn(), 50);
// write data to mds table no.1 row
{
ObTabletCreateDeleteMdsUserData user_data;
user_data.tablet_status_ = ObTabletStatus::NORMAL;
user_data.data_type_ = ObTabletMdsUserDataType::CREATE_TABLET;
mds::MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(123)));
ret = src_handle.get_obj()->set_tablet_status(user_data, ctx);
ASSERT_EQ(OB_SUCCESS, ret);
ctx.single_log_commit(create_commit_scn, create_commit_scn);
}
ObMigrationTabletParam tablet_param;
ret = src_handle.get_obj()->build_migration_tablet_param(tablet_param);
ASSERT_EQ(OB_SUCCESS, ret);

47
mittest/mtlenv/storage/test_ls_tablet_service.cpp
View File

@ -44,6 +44,11 @@ namespace oceanbase
using namespace share::schema;
using namespace share;
using namespace common;
int ObClusterVersion::get_tenant_data_version(const uint64_t tenant_id, uint64_t &data_version)
{
data_version = DATA_VERSION_4_3_2_0;
return OB_SUCCESS;
}
namespace storage
{
@ -630,9 +635,7 @@ TEST_F(TestLSTabletService, test_get_ls_min_end_scn)
ObTabletHandle tablet_handle;
ret = t3m->get_tablet(WashTabletPriority::WTP_HIGH, key, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
share::SCN test_scn = share::SCN::min_scn();
share::SCN expect_scn;
share::SCN orig_scn;
expect_scn.val_ = 0;
share::SCN min_end_scn_from_latest_tablets = SCN::max_scn();
@ -641,17 +644,6 @@ TEST_F(TestLSTabletService, test_get_ls_min_end_scn)
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(min_end_scn_from_latest_tablets, expect_scn);
orig_scn = tablet_handle.get_obj()->tablet_meta_.clog_checkpoint_scn_;
tablet_handle.get_obj()->tablet_meta_.clog_checkpoint_scn_ = test_scn; // modify scn of tablet
min_end_scn_from_latest_tablets.set_max();
min_end_scn_from_old_tablets.set_max();
ret = ls_tablet_service_->get_ls_min_end_scn(min_end_scn_from_latest_tablets, min_end_scn_from_old_tablets);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(min_end_scn_from_latest_tablets, expect_scn); // still get from major sstable
tablet_handle.get_obj()->tablet_meta_.clog_checkpoint_scn_ = orig_scn; // set orig_scn to del tablet
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -756,26 +748,18 @@ TEST_F(TestLSTabletService, test_cover_empty_shell)
ret = ls_svr->get_ls(ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD);
ASSERT_EQ(OB_SUCCESS, ret);
ret = TestTabletHelper::create_tablet(ls_handle, tablet_id, schema, allocator_, ObTabletStatus::Status::DELETED);
share::SCN create_commit_scn = share::SCN::plus(share::SCN::base_scn(), 100);
ret = TestTabletHelper::create_tablet(ls_handle, tablet_id, schema, allocator_, ObTabletStatus::Status::DELETED, create_commit_scn);
ASSERT_EQ(OB_SUCCESS, ret);
ObTabletHandle old_tablet_handle;
ret = ls_handle.get_ls()->get_tablet_svr()->get_tablet(tablet_id, old_tablet_handle, 0, ObMDSGetTabletMode::READ_WITHOUT_CHECK);
ASSERT_EQ(OB_SUCCESS, ret);
ObMigrationTabletParam tablet_meta;
ret = old_tablet_handle.get_obj()->build_migration_tablet_param(tablet_meta);
ObMigrationTabletParam param;
ret = old_tablet_handle.get_obj()->build_migration_tablet_param(param);
ASSERT_EQ(OB_SUCCESS, ret);
ObTabletCreateDeleteMdsUserData data;
ObTabletStatus status(ObTabletStatus::NORMAL);
data.tablet_status_ = status;
const int64_t data_serialize_size = data.get_serialize_size();
int64_t pos = 0;
char *buf = static_cast<char *>(allocator_.alloc(data_serialize_size));
ASSERT_EQ(OB_SUCCESS, data.serialize(buf, data_serialize_size, pos));
tablet_meta.mds_data_.tablet_status_committed_kv_.v_.user_data_.assign_ptr(buf, data_serialize_size);
ret = ls_tablet_service_->update_tablet_to_empty_shell(tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
ObTabletHandle test_tablet_handle;
@ -786,10 +770,9 @@ TEST_F(TestLSTabletService, test_cover_empty_shell)
ASSERT_EQ(ObTabletStatus::DELETED, user_data.tablet_status_);
ObTabletHandle tablet_handle;
ret = ls_tablet_service_->create_transfer_in_tablet(ls_id_, tablet_meta, tablet_handle);
ret = ls_tablet_service_->create_transfer_in_tablet(ls_id_, param, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->get_tablet_status(share::SCN::max_scn(), user_data));
ASSERT_EQ(ObTabletStatus::NORMAL, user_data.tablet_status_);
ASSERT_EQ(OB_EMPTY_RESULT, tablet_handle.get_obj()->get_tablet_status(share::SCN::max_scn(), user_data));
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
@ -990,10 +973,10 @@ TEST_F(TestLSTabletService, test_migrate_param_empty_shell)
ObMigrationTabletParam other_tablet_meta;
ASSERT_EQ(OB_SUCCESS, other_tablet_meta.assign(de_tablet_meta));
/*
ObMigrationTabletParam other_tablet_meta;
ASSERT_EQ(OB_SUCCESS, tablet_meta.build_deleted_tablet_info(ls_id_, tablet_id));
*/
//ObMigrationTabletParam other_tablet_meta;
//ASSERT_EQ(OB_SUCCESS, tablet_meta.build_deleted_tablet_info(ls_id_, tablet_id));
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);

463
mittest/mtlenv/storage/test_mds_compat.cpp Normal file
View File

@ -0,0 +1,463 @@
/**
* Copyright (c) 2024 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#define protected public
#define private public
#include "lib/ob_errno.h"
#include "lib/allocator/page_arena.h"
#include "lib/oblog/ob_log.h"
#include "common/ob_tablet_id.h"
#include "share/ob_ls_id.h"
#include "share/scn.h"
#include "share/rc/ob_tenant_base.h"
#include "mtlenv/mock_tenant_module_env.h"
#include "mtlenv/storage/medium_info_helper.h"
#include "unittest/storage/test_tablet_helper.h"
#include "unittest/storage/test_dml_common.h"
#include "storage/tablet/ob_mds_row_iterator.h"
#include "storage/tablet/ob_tablet_mds_table_mini_merger.h"
#define USING_LOG_PREFIX STORAGE
using namespace oceanbase::common;
using namespace oceanbase::share;
namespace oceanbase
{
namespace storage
{
class TestMdsCompat : public::testing::Test
{
public:
TestMdsCompat() = default;
virtual ~TestMdsCompat() = default;
public:
static void SetUpTestCase();
static void TearDownTestCase();
public:
static int create_ls(const uint64_t tenant_id, const share::ObLSID &ls_id, ObLSHandle &ls_handle);
static int remove_ls(const share::ObLSID &ls_id);
int create_tablet(const common::ObTabletID &tablet_id, ObTabletHandle &tablet_handle);
static int get_tablet(const common::ObTabletID &tablet_id, ObTabletHandle &tablet_handle);
static int wait_for_mds_table_flush(const common::ObTabletID &tablet_id);
static int wait_for_all_mds_nodes_released(const common::ObTabletID &tablet_id);
static int try_schedule_mds_minor(const common::ObTabletID &tablet_id);
public:
static constexpr uint64_t TENANT_ID = 1001;
static const share::ObLSID LS_ID;
public:
common::ObArenaAllocator allocator_;
};
const share::ObLSID TestMdsCompat::LS_ID(1234);
void TestMdsCompat::SetUpTestCase()
{
int ret = OB_SUCCESS;
ret = MockTenantModuleEnv::get_instance().init();
ASSERT_EQ(OB_SUCCESS, ret);
ObServerCheckpointSlogHandler::get_instance().is_started_ = true;
// create ls
ObLSHandle ls_handle;
ret = create_ls(TENANT_ID, LS_ID, ls_handle);
ASSERT_EQ(OB_SUCCESS, ret);
}
void TestMdsCompat::TearDownTestCase()
{
int ret = OB_SUCCESS;
// remove ls
ret = remove_ls(LS_ID);
ASSERT_EQ(OB_SUCCESS, ret);
MockTenantModuleEnv::get_instance().destroy();
}
int TestMdsCompat::create_ls(const uint64_t tenant_id, const share::ObLSID &ls_id, ObLSHandle &ls_handle)
{
int ret = OB_SUCCESS;
ret = TestDmlCommon::create_ls(tenant_id, ls_id, ls_handle);
return ret;
}
int TestMdsCompat::remove_ls(const share::ObLSID &ls_id)
{
int ret = OB_SUCCESS;
ret = MTL(ObLSService*)->remove_ls(ls_id);
return ret;
}
int TestMdsCompat::create_tablet(const common::ObTabletID &tablet_id, ObTabletHandle &tablet_handle)
{
int ret = OB_SUCCESS;
const uint64_t table_id = 1234567;
share::schema::ObTableSchema table_schema;
ObLSHandle ls_handle;
ObLS *ls = nullptr;
if (OB_FAIL(MTL(ObLSService*)->get_ls(LS_ID, ls_handle, ObLSGetMod::STORAGE_MOD))) {
LOG_WARN("failed to get ls", K(ret));
} else if (OB_ISNULL(ls = ls_handle.get_ls())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls is null", K(ret), KP(ls));
} else if (OB_FAIL(build_test_schema(table_schema, table_id))) {
LOG_WARN("failed to build table schema");
} else if (OB_FAIL(TestTabletHelper::create_tablet(ls_handle, tablet_id, table_schema, allocator_,
ObTabletStatus::MAX, share::SCN::invalid_scn(), tablet_handle))) {
LOG_WARN("failed to create tablet", K(ret));
}
return ret;
}
int TestMdsCompat::get_tablet(const common::ObTabletID &tablet_id, ObTabletHandle &tablet_handle)
{
int ret = OB_SUCCESS;
ObLSHandle ls_handle;
ObLS *ls = nullptr;
if (OB_FAIL(MTL(ObLSService*)->get_ls(LS_ID, ls_handle, ObLSGetMod::STORAGE_MOD))) {
LOG_WARN("failed to get ls", K(ret));
} else if (OB_ISNULL(ls = ls_handle.get_ls())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls is null", K(ret), KP(ls));
} else if (OB_FAIL(ls->get_tablet_svr()->direct_get_tablet(tablet_id, tablet_handle))) {
LOG_WARN("failed to get tablet", K(ret), KP(ls));
}
return ret;
}
int TestMdsCompat::wait_for_mds_table_flush(const common::ObTabletID &tablet_id)
{
int ret = OB_SUCCESS;
ObTabletHandle tablet_handle;
ObTablet *tablet = nullptr;
int times = 0;
share::SCN rec_scn = share::SCN::min_scn();
// get before cnt
ret = TestMdsCompat::get_tablet(tablet_id, tablet_handle);
EXPECT_EQ(OB_SUCCESS, ret);
tablet = tablet_handle.get_obj();
EXPECT_NE(nullptr, tablet);
ObTabletMemberWrapper<ObTabletTableStore> table_store_wrapper;
ret = tablet->fetch_table_store(table_store_wrapper);
EXPECT_EQ(OB_SUCCESS, ret);
const ObTabletTableStore *table_store = table_store_wrapper.get_member();
const int64_t mds_sstable_cnt_before = table_store->mds_sstables_.count();
do
{
ret = TestMdsCompat::get_tablet(tablet_id, tablet_handle);
EXPECT_EQ(OB_SUCCESS, ret);
tablet = tablet_handle.get_obj();
EXPECT_NE(nullptr, tablet);
mds::MdsTableHandle mds_table;
ret = tablet->inner_get_mds_table(mds_table, false/*not_exist_create*/);
ret = mds_table.get_rec_scn(rec_scn);
EXPECT_EQ(OB_SUCCESS, ret);
// sleep
::ob_usleep(100_ms);
++times;
} while (OB_SUCCESS == ret && !rec_scn.is_max() && times < 20);
EXPECT_TRUE(rec_scn.is_max());
// check mds sstable
ret = tablet->fetch_table_store(table_store_wrapper);
EXPECT_EQ(OB_SUCCESS, ret);
table_store = table_store_wrapper.get_member();
EXPECT_EQ(mds_sstable_cnt_before + 1, table_store->mds_sstables_.count());
if (::testing::Test::HasFailure()) {
ret = OB_TIMEOUT;
}
return ret;
}
int TestMdsCompat::wait_for_all_mds_nodes_released(const common::ObTabletID &tablet_id)
{
int ret = OB_SUCCESS;
ObTabletHandle tablet_handle;
ObTablet *tablet = nullptr;
int times = 0;
int64_t node_cnt = INT64_MAX;
do {
ret = TestMdsCompat::get_tablet(tablet_id, tablet_handle);
EXPECT_EQ(OB_SUCCESS, ret);
tablet = tablet_handle.get_obj();
EXPECT_NE(nullptr, tablet);
mds::MdsTableHandle mds_table;
ret = tablet->inner_get_mds_table(mds_table, false/*not_exist_create*/);
EXPECT_EQ(OB_SUCCESS, ret);
ret = mds_table.get_node_cnt(node_cnt);
EXPECT_EQ(OB_SUCCESS, ret);
// sleep
::ob_usleep(100_ms);
++times;
} while (OB_SUCCESS == ret && node_cnt != 0 && times < 60);
EXPECT_EQ(0, node_cnt);
if (::testing::Test::HasFailure()) {
ret = OB_TIMEOUT;
}
return ret;
}
TEST_F(TestMdsCompat, migration_param)
{
int ret = OB_SUCCESS;
// create tablet
const common::ObTabletID tablet_id(ObTimeUtility::fast_current_time() % 10000000000000);
ObTabletHandle tablet_handle;
ObTabletStatus status(ObTabletStatus::MAX);
share::SCN create_commit_scn;
create_commit_scn = share::SCN::plus(share::SCN::min_scn(), 50);
ret = create_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *tablet = tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
share::SCN mds_checkpoint_scn = tablet->get_tablet_meta().mds_checkpoint_scn_;
ASSERT_EQ(share::SCN::base_scn(), mds_checkpoint_scn);
share::SCN invalid_scn;
// write data to mds table no.1 row
{
ObTabletCreateDeleteMdsUserData user_data;
user_data.tablet_status_ = ObTabletStatus::NORMAL;
user_data.data_type_ = ObTabletMdsUserDataType::CREATE_TABLET;
mds::MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(123)));
ret = tablet->set_tablet_status(user_data, ctx);
ASSERT_EQ(OB_SUCCESS, ret);
ctx.single_log_commit(create_commit_scn, create_commit_scn);
}
// write data to mds table no.2 row
{
compaction::ObMediumCompactionInfoKey key(100);
compaction::ObMediumCompactionInfo info;
ret = MediumInfoHelper::build_medium_compaction_info(allocator_, info, 100);
ASSERT_EQ(OB_SUCCESS, ret);
mds::MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(777)));
ret = tablet->set(key, info, ctx, 1_s/*lock_timeout_us*/);
ASSERT_EQ(OB_SUCCESS, ret);
share::SCN redo_scn = share::SCN::plus(share::SCN::min_scn(), 110);
ctx.on_redo(redo_scn);
share::SCN commit_scn = share::SCN::plus(share::SCN::min_scn(), 120);
ctx.on_commit(commit_scn, commit_scn);
}
{
compaction::ObMediumCompactionInfoKey key(200);
compaction::ObMediumCompactionInfo info;
ret = MediumInfoHelper::build_medium_compaction_info(allocator_, info, 200);
ASSERT_EQ(OB_SUCCESS, ret);
mds::MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(888)));
ret = tablet->set(key, info, ctx, 1_s/*lock_timeout_us*/);
ASSERT_EQ(OB_SUCCESS, ret);
share::SCN redo_scn = share::SCN::plus(share::SCN::min_scn(), 210);
ctx.on_redo(redo_scn);
share::SCN commit_scn = share::SCN::plus(share::SCN::min_scn(), 220);
ctx.on_commit(commit_scn, commit_scn);
}
// write data to mds table no.3 row
{
ObTabletBindingMdsUserData user_data;
user_data.data_tablet_id_ = 100;
user_data.hidden_tablet_id_ = 101;
user_data.snapshot_version_ = 9527;
mds::MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(1000)));
ret = tablet->set_ddl_info(user_data, ctx, 1_s/*lock_timeout_us*/);
ASSERT_EQ(OB_SUCCESS, ret);
share::SCN redo_scn = share::SCN::plus(share::SCN::min_scn(), 250);
ctx.on_redo(redo_scn);
share::SCN commit_scn = share::SCN::plus(share::SCN::min_scn(), 280);
ctx.on_commit(commit_scn, commit_scn);
}
{
share::ObTabletAutoincSeq user_data;
mds::MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(1200)));
ret = tablet->set(user_data, ctx, 1_s/*lock_timeout_us*/);
ASSERT_EQ(OB_SUCCESS, ret);
share::SCN redo_scn = share::SCN::plus(share::SCN::min_scn(), 300);
ctx.on_redo(redo_scn);
share::SCN commit_scn = share::SCN::plus(share::SCN::min_scn(), 320);
ctx.on_commit(commit_scn, commit_scn);
}
// mock mds data
ObTabletMdsData mds_table_data;
ObTabletMdsData base_data;
base_data.init_for_first_creation();
ObTabletMdsData mocked_mds_data;
ret = tablet->read_mds_table(allocator_, mds_table_data, false/*for_flush*/);
ASSERT_EQ(OB_SUCCESS, ret);
LOG_INFO("read mds table", K(ret), K(mds_table_data));
ret = mocked_mds_data.init_for_mds_table_dump(allocator_, mds_table_data, base_data, 0/*finish_medium_scn*/);
ASSERT_EQ(OB_SUCCESS, ret);
// mds table flush
share::SCN decided_scn;
decided_scn = share::SCN::plus(share::SCN::min_scn(), 410);
ret = tablet->mds_table_flush(decided_scn);
ASSERT_EQ(OB_SUCCESS, ret);
// wait for mds table flush
ret = wait_for_mds_table_flush(tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
// wait all mds nodes to be released
tablet_handle.reset();
ret = wait_for_all_mds_nodes_released(tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
ret = TestMdsCompat::get_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
tablet = tablet_handle.get_obj();
ObMigrationTabletParam param;
ret = tablet->build_migration_tablet_param(param);
param.last_persisted_committed_tablet_status_.on_init();
ASSERT_EQ(OB_NOT_SUPPORTED, ret);
ASSERT_TRUE(param.is_valid());
// replace mds data in migration tablet param with our mocked mds data
param.mds_data_.reset();
ret = param.mds_data_.init(param.allocator_, mocked_mds_data);
ASSERT_EQ(OB_SUCCESS, ret);
LOG_INFO("start generate mds sstable from param", K(ret), K(param));
ObTableHandleV2 table_handle;
ret = ObMdsDataCompatHelper::generate_mds_mini_sstable(param, allocator_, table_handle);
ASSERT_EQ(common::OB_SUCCESS, ret);
const ObITable *table = table_handle.get_table();
ASSERT_NE(nullptr, table);
ASSERT_EQ(share::SCN::plus(share::SCN::min_scn(), 1), table->get_start_scn());
ASSERT_EQ(param.mds_checkpoint_scn_, table->get_end_scn());
}
TEST_F(TestMdsCompat, compat)
{
int ret = OB_SUCCESS;
// create tablet
const common::ObTabletID tablet_id(ObTimeUtility::fast_current_time() % 10000000000000);
ObTabletHandle tablet_handle;
share::SCN create_commit_scn;
create_commit_scn = share::SCN::plus(share::SCN::min_scn(), 80);
ret = create_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *tablet = tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
share::SCN mds_checkpoint_scn = tablet->get_tablet_meta().mds_checkpoint_scn_;
ASSERT_EQ(share::SCN::base_scn(), mds_checkpoint_scn);
// write data to mds table
{
ObTabletCreateDeleteMdsUserData user_data;
user_data.tablet_status_ = ObTabletStatus::NORMAL;
user_data.data_type_ = ObTabletMdsUserDataType::CREATE_TABLET;
mds::MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(123)));
ret = tablet->set_tablet_status(user_data, ctx);
ASSERT_EQ(OB_SUCCESS, ret);
share::SCN redo_scn = share::SCN::plus(share::SCN::min_scn(), 50);
ctx.on_redo(redo_scn);
ctx.on_commit(create_commit_scn, create_commit_scn);
}
{
compaction::ObMediumCompactionInfoKey key(100);
compaction::ObMediumCompactionInfo info;
ret = MediumInfoHelper::build_medium_compaction_info(allocator_, info, 100);
ASSERT_EQ(OB_SUCCESS, ret);
mds::MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(777)));
ret = tablet->set(key, info, ctx, 1_s/*lock_timeout_us*/);
ASSERT_EQ(OB_SUCCESS, ret);
share::SCN redo_scn = share::SCN::plus(share::SCN::min_scn(), 110);
ctx.on_redo(redo_scn);
share::SCN commit_scn = share::SCN::plus(share::SCN::min_scn(), 120);
ctx.on_commit(commit_scn, commit_scn);
}
// mock mds data
ObTabletMdsData mds_table_data;
ObTabletMdsData base_data;
base_data.init_for_first_creation();
ObTabletMdsData mocked_mds_data;
ret = tablet->read_mds_table(allocator_, mds_table_data, false/*for_flush*/);
ASSERT_EQ(OB_SUCCESS, ret);
LOG_INFO("read mds table", K(ret), K(mds_table_data));
ret = mocked_mds_data.init_for_mds_table_dump(allocator_, mds_table_data, base_data, 0/*finish_medium_scn*/);
ASSERT_EQ(OB_SUCCESS, ret);
// assign to tablet
ASSERT_EQ(nullptr, tablet->mds_data_);
ret = ObTabletObjLoadHelper::alloc_and_new(allocator_, tablet->mds_data_);
ASSERT_NE(nullptr, tablet->mds_data_);
ret = tablet->mds_data_->init_for_evict_medium_info(allocator_, mocked_mds_data, 0/*finish_medium_scn*/);
ASSERT_EQ(OB_SUCCESS, ret);
// convert to mds sstable
ObArenaAllocator allocator;
ObTableHandleV2 table_handle;
ret = ObMdsDataCompatHelper::generate_mds_mini_sstable(*tablet_handle.get_obj(), allocator, table_handle);
ASSERT_EQ(OB_SUCCESS, ret);
blocksstable::ObSSTable *sstable = nullptr;
ASSERT_EQ(OB_SUCCESS, table_handle.get_sstable(sstable));
ASSERT_NE(nullptr, sstable);
ASSERT_TRUE(sstable->is_valid());
}
} // namespace storage
} // namespace oceanbase
int main(int argc, char **argv)
{
system("rm -f test_mds_compat.log*");
OB_LOGGER.set_file_name("test_mds_compat.log", true);
OB_LOGGER.set_log_level("INFO");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

1199
mittest/mtlenv/storage/test_mds_table_scan.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

566
mittest/mtlenv/storage/test_medium_info_iterator.cpp Normal file
View File

@ -0,0 +1,566 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#define private public
#define protected public
#include "mtlenv/storage/medium_info_common.h"
#include "storage/tablet/ob_mds_scan_param_helper.h"
#include "storage/tablet/ob_mds_range_query_iterator.h"
using namespace oceanbase::common;
using namespace oceanbase::share;
using namespace oceanbase::unittest;
namespace oceanbase
{
namespace storage
{
class TestMediumInfoIterator : public MediumInfoCommon
{
public:
TestMediumInfoIterator() = default;
virtual ~TestMediumInfoIterator() = default;
};
TEST_F(TestMediumInfoIterator, pure_mds_table)
{
int ret = OB_SUCCESS;
// create tablet
const common::ObTabletID tablet_id(ObTimeUtility::fast_current_time() % 10000000000000);
ObTabletHandle tablet_handle;
ret = create_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *tablet = tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
{
ObTabletCreateDeleteMdsUserData user_data;
user_data.tablet_status_ = ObTabletStatus::NORMAL;
user_data.data_type_ = ObTabletMdsUserDataType::CREATE_TABLET;
share::SCN commit_scn = share::SCN::plus(share::SCN::min_scn(), 5);
user_data.create_commit_scn_ = commit_scn;
user_data.create_commit_version_ = 5;
mds::MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(5)));
ret = tablet->set_tablet_status(user_data, ctx);
ASSERT_EQ(OB_SUCCESS, ret);
ctx.single_log_commit(commit_scn, commit_scn);
}
// insert data into mds table
ret = insert_medium_info(10, 10);
ASSERT_EQ(OB_SUCCESS, ret);
ret = insert_medium_info(20, 20);
ASSERT_EQ(OB_SUCCESS, ret);
// query
common::ObArenaAllocator arena_allocator;
constexpr uint8_t mds_unit_id = mds::TupleTypeIdx<mds::NormalMdsTable, mds::MdsUnit<compaction::ObMediumCompactionInfoKey, compaction::ObMediumCompactionInfo>>::value;
ObTableScanParam scan_param;
share::SCN read_snapshot = share::SCN::plus(share::SCN::min_scn(), 30);
ret = ObMdsScanParamHelper::build_scan_param(
arena_allocator,
LS_ID,
tablet_id,
ObMdsSchemaHelper::MDS_TABLE_ID,
mds_unit_id,
common::ObString()/*udf_key*/,
false/*is_get*/,
ObClockGenerator::getClock() + 1_s,
read_snapshot,
scan_param);
ASSERT_EQ(OB_SUCCESS, ret);
ObStoreCtx store_ctx;
ObMdsRangeQueryIterator<compaction::ObMediumCompactionInfoKey, compaction::ObMediumCompactionInfo> range_query_iter;
ret = tablet->mds_range_query<compaction::ObMediumCompactionInfoKey, compaction::ObMediumCompactionInfo>(
scan_param, store_ctx, range_query_iter);
ASSERT_EQ(OB_SUCCESS, ret);
mds::MdsDumpKV *kv = nullptr;
{
ret = range_query_iter.get_next_mds_kv(allocator_, kv);
ASSERT_EQ(OB_SUCCESS, ret);
compaction::ObMediumCompactionInfoKey key;
const common::ObString &str = kv->k_.key_;
int64_t pos = 0;
ret = key.mds_deserialize(str.ptr(), str.length(), pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(10, key.medium_snapshot_);
range_query_iter.free_mds_kv(allocator_, kv);
}
{
ret = range_query_iter.get_next_mds_kv(allocator_, kv);
ASSERT_EQ(OB_SUCCESS, ret);
compaction::ObMediumCompactionInfoKey key;
const common::ObString &str = kv->k_.key_;
int64_t pos = 0;
ret = key.mds_deserialize(str.ptr(), str.length(), pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(20, key.medium_snapshot_);
range_query_iter.free_mds_kv(allocator_, kv);
}
{
ret = range_query_iter.get_next_mds_kv(allocator_, kv);
ASSERT_EQ(OB_ITER_END, ret);
}
}
TEST_F(TestMediumInfoIterator, pure_mds_sstable)
{
int ret = OB_SUCCESS;
// get ls
ObLSHandle ls_handle;
ObLS *ls = nullptr;
ret = MTL(ObLSService*)->get_ls(LS_ID, ls_handle, ObLSGetMod::STORAGE_MOD);
ASSERT_EQ(OB_SUCCESS, ret);
ls = ls_handle.get_ls();
ASSERT_NE(nullptr, ls);
// create tablet
const common::ObTabletID tablet_id(ObTimeUtility::fast_current_time() % 10000000000000);
ObTabletHandle tablet_handle;
ret = create_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *tablet = tablet_handle.get_obj();
{
ObTabletCreateDeleteMdsUserData user_data;
user_data.tablet_status_ = ObTabletStatus::NORMAL;
user_data.data_type_ = ObTabletMdsUserDataType::CREATE_TABLET;
share::SCN commit_scn = share::SCN::plus(share::SCN::min_scn(), 5);
user_data.create_commit_scn_ = commit_scn;
user_data.create_commit_version_ = 5;
mds::MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(5)));
ret = tablet->set_tablet_status(user_data, ctx);
ASSERT_EQ(OB_SUCCESS, ret);
ctx.single_log_commit(commit_scn, commit_scn);
}
// insert data into mds table
ret = insert_medium_info(10, 10);
ASSERT_EQ(OB_SUCCESS, ret);
ret = insert_medium_info(20, 20);
ASSERT_EQ(OB_SUCCESS, ret);
ret = insert_medium_info(30, 30);
ASSERT_EQ(OB_SUCCESS, ret);
// mds table flush
share::SCN decided_scn = share::SCN::plus(share::SCN::min_scn(), 50);
ret = tablet->mds_table_flush(decided_scn);
ASSERT_EQ(OB_SUCCESS, ret);
// wait for mds table flush
tablet_handle.reset();
ret = wait_for_mds_table_flush(tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
// wait all mds nodes to be released
ret = wait_for_all_mds_nodes_released(tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
// query
ret = ls->get_tablet_svr()->direct_get_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
tablet = tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
// query
common::ObArenaAllocator arena_allocator;
constexpr uint8_t mds_unit_id = mds::TupleTypeIdx<mds::NormalMdsTable, mds::MdsUnit<compaction::ObMediumCompactionInfoKey, compaction::ObMediumCompactionInfo>>::value;
ObTableScanParam scan_param;
share::SCN read_snapshot = share::SCN::plus(share::SCN::min_scn(), 22);
ret = ObMdsScanParamHelper::build_scan_param(
arena_allocator,
LS_ID,
tablet_id,
ObMdsSchemaHelper::MDS_TABLE_ID,
mds_unit_id,
common::ObString()/*udf_key*/,
false/*is_get*/,
ObClockGenerator::getClock() + 1_s,
read_snapshot,
scan_param);
ASSERT_EQ(OB_SUCCESS, ret);
ObStoreCtx store_ctx;
ObMdsRangeQueryIterator<compaction::ObMediumCompactionInfoKey, compaction::ObMediumCompactionInfo> range_query_iter;
ret = tablet->mds_range_query<compaction::ObMediumCompactionInfoKey, compaction::ObMediumCompactionInfo>(
scan_param, store_ctx, range_query_iter);
ASSERT_EQ(OB_SUCCESS, ret);
mds::MdsDumpKV *kv = nullptr;
{
ret = range_query_iter.get_next_mds_kv(allocator_, kv);
ASSERT_EQ(OB_SUCCESS, ret);
compaction::ObMediumCompactionInfoKey key;
const common::ObString &str = kv->k_.key_;
int64_t pos = 0;
ret = key.mds_deserialize(str.ptr(), str.length(), pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(10, key.medium_snapshot_);
range_query_iter.free_mds_kv(allocator_, kv);
}
{
ret = range_query_iter.get_next_mds_kv(allocator_, kv);
ASSERT_EQ(OB_SUCCESS, ret);
compaction::ObMediumCompactionInfoKey key;
const common::ObString &str = kv->k_.key_;
int64_t pos = 0;
ret = key.mds_deserialize(str.ptr(), str.length(), pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(20, key.medium_snapshot_);
range_query_iter.free_mds_kv(allocator_, kv);
}
{
ret = range_query_iter.get_next_mds_kv(allocator_, kv);
ASSERT_EQ(OB_ITER_END, ret);
}
}
TEST_F(TestMediumInfoIterator, data_overlap)
{
}
TEST_F(TestMediumInfoIterator, data_no_overlap)
{
int ret = OB_SUCCESS;
// get ls
ObLSHandle ls_handle;
ObLS *ls = nullptr;
ret = MTL(ObLSService*)->get_ls(LS_ID, ls_handle, ObLSGetMod::STORAGE_MOD);
ASSERT_EQ(OB_SUCCESS, ret);
ls = ls_handle.get_ls();
ASSERT_NE(nullptr, ls);
// create tablet
const common::ObTabletID tablet_id(ObTimeUtility::fast_current_time() % 10000000000000);
ObTabletHandle tablet_handle;
ret = create_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *tablet = tablet_handle.get_obj();
{
ObTabletCreateDeleteMdsUserData user_data;
user_data.tablet_status_ = ObTabletStatus::NORMAL;
user_data.data_type_ = ObTabletMdsUserDataType::CREATE_TABLET;
share::SCN commit_scn = share::SCN::plus(share::SCN::min_scn(), 5);
user_data.create_commit_scn_ = commit_scn;
user_data.create_commit_version_ = 5;
mds::MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(5)));
ret = tablet->set_tablet_status(user_data, ctx);
ASSERT_EQ(OB_SUCCESS, ret);
ctx.single_log_commit(commit_scn, commit_scn);
}
// insert data into mds table
ret = insert_medium_info(100, 100);
ASSERT_EQ(OB_SUCCESS, ret);
ret = insert_medium_info(200, 200);
ASSERT_EQ(OB_SUCCESS, ret);
ret = insert_medium_info(300, 300);
ASSERT_EQ(OB_SUCCESS, ret);
// mds table flush
share::SCN decided_scn = share::SCN::plus(share::SCN::min_scn(), 310);
ret = tablet->mds_table_flush(decided_scn);
ASSERT_EQ(OB_SUCCESS, ret);
// wait for mds table flush
tablet_handle.reset();
ret = wait_for_mds_table_flush(tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
// wait all mds nodes to be released
ret = wait_for_all_mds_nodes_released(tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
// insert data into mds table
ret = insert_medium_info(400, 400);
ASSERT_EQ(OB_SUCCESS, ret);
ret = insert_medium_info(500, 500);
ASSERT_EQ(OB_SUCCESS, ret);
// query
ret = ls->get_tablet_svr()->direct_get_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
tablet = tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
common::ObArenaAllocator arena_allocator;
constexpr uint8_t mds_unit_id = mds::TupleTypeIdx<mds::NormalMdsTable, mds::MdsUnit<compaction::ObMediumCompactionInfoKey, compaction::ObMediumCompactionInfo>>::value;
ObTableScanParam scan_param;
share::SCN read_snapshot = share::SCN::plus(share::SCN::min_scn(), 444);
ret = ObMdsScanParamHelper::build_scan_param(
arena_allocator,
LS_ID,
tablet_id,
ObMdsSchemaHelper::MDS_TABLE_ID,
mds_unit_id,
common::ObString()/*udf_key*/,
false/*is_get*/,
ObClockGenerator::getClock() + 1_s,
read_snapshot,
scan_param);
ASSERT_EQ(OB_SUCCESS, ret);
ObStoreCtx store_ctx;
ObMdsRangeQueryIterator<compaction::ObMediumCompactionInfoKey, compaction::ObMediumCompactionInfo> range_query_iter;
ret = tablet->mds_range_query<compaction::ObMediumCompactionInfoKey, compaction::ObMediumCompactionInfo>(
scan_param, store_ctx, range_query_iter);
ASSERT_EQ(OB_SUCCESS, ret);
mds::MdsDumpKV *kv = nullptr;
{
ret = range_query_iter.get_next_mds_kv(allocator_, kv);
ASSERT_EQ(OB_SUCCESS, ret);
compaction::ObMediumCompactionInfoKey key;
const common::ObString &str = kv->k_.key_;
int64_t pos = 0;
ret = key.mds_deserialize(str.ptr(), str.length(), pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(100, key.medium_snapshot_);
range_query_iter.free_mds_kv(allocator_, kv);
}
{
ret = range_query_iter.get_next_mds_kv(allocator_, kv);
ASSERT_EQ(OB_SUCCESS, ret);
compaction::ObMediumCompactionInfoKey key;
const common::ObString &str = kv->k_.key_;
int64_t pos = 0;
ret = key.mds_deserialize(str.ptr(), str.length(), pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(200, key.medium_snapshot_);
range_query_iter.free_mds_kv(allocator_, kv);
}
{
ret = range_query_iter.get_next_mds_kv(allocator_, kv);
ASSERT_EQ(OB_SUCCESS, ret);
compaction::ObMediumCompactionInfoKey key;
const common::ObString &str = kv->k_.key_;
int64_t pos = 0;
ret = key.mds_deserialize(str.ptr(), str.length(), pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(300, key.medium_snapshot_);
range_query_iter.free_mds_kv(allocator_, kv);
}
{
ret = range_query_iter.get_next_mds_kv(allocator_, kv);
ASSERT_EQ(OB_SUCCESS, ret);
compaction::ObMediumCompactionInfoKey key;
const common::ObString &str = kv->k_.key_;
int64_t pos = 0;
ret = key.mds_deserialize(str.ptr(), str.length(), pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(400, key.medium_snapshot_);
range_query_iter.free_mds_kv(allocator_, kv);
}
{
ret = range_query_iter.get_next_mds_kv(allocator_, kv);
ASSERT_EQ(OB_ITER_END, ret);
}
}
TEST_F(TestMediumInfoIterator, full_inclusion)
{
int ret = OB_SUCCESS;
// get ls
ObLSHandle ls_handle;
ObLS *ls = nullptr;
ret = MTL(ObLSService*)->get_ls(LS_ID, ls_handle, ObLSGetMod::STORAGE_MOD);
ASSERT_EQ(OB_SUCCESS, ret);
ls = ls_handle.get_ls();
ASSERT_NE(nullptr, ls);
// create tablet
const common::ObTabletID tablet_id(ObTimeUtility::fast_current_time() % 10000000000000);
ObTabletHandle tablet_handle;
ret = create_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *tablet = tablet_handle.get_obj();
{
ObTabletCreateDeleteMdsUserData user_data;
user_data.tablet_status_ = ObTabletStatus::NORMAL;
user_data.data_type_ = ObTabletMdsUserDataType::CREATE_TABLET;
share::SCN commit_scn = share::SCN::plus(share::SCN::min_scn(), 5);
user_data.create_commit_scn_ = commit_scn;
user_data.create_commit_version_ = 5;
mds::MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(5)));
ret = tablet->set_tablet_status(user_data, ctx);
ASSERT_EQ(OB_SUCCESS, ret);
ctx.single_log_commit(commit_scn, commit_scn);
}
// insert data into mds table
ret = insert_medium_info(10, 10);
ASSERT_EQ(OB_SUCCESS, ret);
ret = insert_medium_info(20, 20);
ASSERT_EQ(OB_SUCCESS, ret);
ret = insert_medium_info(30, 30);
ASSERT_EQ(OB_SUCCESS, ret);
// mds table flush
share::SCN decided_scn = share::SCN::plus(share::SCN::min_scn(), 31);
ret = tablet->mds_table_flush(decided_scn);
ASSERT_EQ(OB_SUCCESS, ret);
// wait for mds table flush
ret = wait_for_mds_table_flush(tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
// nodes are not released
int64_t node_cnt = INT64_MAX;
::ob_usleep(6_s);
ret = mds_table_.get_node_cnt(node_cnt);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(4, node_cnt);
// insert data into mds table
ret = insert_medium_info(40, 40);
ASSERT_EQ(OB_SUCCESS, ret);
ret = insert_medium_info(50, 50);
ASSERT_EQ(OB_SUCCESS, ret);
// query
ret = ls->get_tablet_svr()->direct_get_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
tablet = tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
common::ObArenaAllocator arena_allocator;
constexpr uint8_t mds_unit_id = mds::TupleTypeIdx<mds::NormalMdsTable, mds::MdsUnit<compaction::ObMediumCompactionInfoKey, compaction::ObMediumCompactionInfo>>::value;
ObTableScanParam scan_param;
share::SCN read_snapshot = share::SCN::plus(share::SCN::min_scn(), 60);
ret = ObMdsScanParamHelper::build_scan_param(
arena_allocator,
LS_ID,
tablet_id,
ObMdsSchemaHelper::MDS_TABLE_ID,
mds_unit_id,
common::ObString()/*udf_key*/,
false/*is_get*/,
ObClockGenerator::getClock() + 1_s,
read_snapshot,
scan_param);
ASSERT_EQ(OB_SUCCESS, ret);
ObStoreCtx store_ctx;
ObMdsRangeQueryIterator<compaction::ObMediumCompactionInfoKey, compaction::ObMediumCompactionInfo> range_query_iter;
ret = tablet->mds_range_query<compaction::ObMediumCompactionInfoKey, compaction::ObMediumCompactionInfo>(
scan_param, store_ctx, range_query_iter);
ASSERT_EQ(OB_SUCCESS, ret);
mds::MdsDumpKV *kv = nullptr;
{
ret = range_query_iter.get_next_mds_kv(allocator_, kv);
ASSERT_EQ(OB_SUCCESS, ret);
compaction::ObMediumCompactionInfoKey key;
const common::ObString &str = kv->k_.key_;
int64_t pos = 0;
ret = key.mds_deserialize(str.ptr(), str.length(), pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(10, key.medium_snapshot_);
range_query_iter.free_mds_kv(allocator_, kv);
}
{
ret = range_query_iter.get_next_mds_kv(allocator_, kv);
ASSERT_EQ(OB_SUCCESS, ret);
compaction::ObMediumCompactionInfoKey key;
const common::ObString &str = kv->k_.key_;
int64_t pos = 0;
ret = key.mds_deserialize(str.ptr(), str.length(), pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(20, key.medium_snapshot_);
range_query_iter.free_mds_kv(allocator_, kv);
}
{
ret = range_query_iter.get_next_mds_kv(allocator_, kv);
ASSERT_EQ(OB_SUCCESS, ret);
compaction::ObMediumCompactionInfoKey key;
const common::ObString &str = kv->k_.key_;
int64_t pos = 0;
ret = key.mds_deserialize(str.ptr(), str.length(), pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(30, key.medium_snapshot_);
range_query_iter.free_mds_kv(allocator_, kv);
}
{
ret = range_query_iter.get_next_mds_kv(allocator_, kv);
ASSERT_EQ(OB_SUCCESS, ret);
compaction::ObMediumCompactionInfoKey key;
const common::ObString &str = kv->k_.key_;
int64_t pos = 0;
ret = key.mds_deserialize(str.ptr(), str.length(), pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(40, key.medium_snapshot_);
range_query_iter.free_mds_kv(allocator_, kv);
}
{
ret = range_query_iter.get_next_mds_kv(allocator_, kv);
ASSERT_EQ(OB_SUCCESS, ret);
compaction::ObMediumCompactionInfoKey key;
const common::ObString &str = kv->k_.key_;
int64_t pos = 0;
ret = key.mds_deserialize(str.ptr(), str.length(), pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(50, key.medium_snapshot_);
range_query_iter.free_mds_kv(allocator_, kv);
}
{
ret = range_query_iter.get_next_mds_kv(allocator_, kv);
ASSERT_EQ(OB_ITER_END, ret);
}
}
} // namespace storage
} // namespace oceanbase
int main(int argc, char **argv)
{
system("rm -f test_medium_info_iterator.log*");
OB_LOGGER.set_file_name("test_medium_info_iterator.log", true);
OB_LOGGER.set_log_level("INFO");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

836
mittest/mtlenv/storage/test_medium_info_reader.cpp
View File

@ -10,776 +10,150 @@
* See the Mulan PubL v2 for more details.
*/
#include "lib/ob_errno.h"
#include <gtest/gtest.h>
#define private public
#define protected public
#include "lib/oblog/ob_log.h"
#include "lib/allocator/page_arena.h"
#include "share/ob_ls_id.h"
#include "common/ob_tablet_id.h"
#include "mtlenv/mock_tenant_module_env.h"
#include "mtlenv/storage/medium_info_helper.h"
#include "share/rc/ob_tenant_base.h"
#include "unittest/storage/test_tablet_helper.h"
#include "unittest/storage/test_dml_common.h"
#include "unittest/storage/init_basic_struct.h"
#include "unittest/storage/schema_utils.h"
#include "storage/tablet/ob_tablet_medium_info_reader.h"
#include "storage/tablet/ob_tablet_persister.h"
#include "storage/multi_data_source/mds_table_handler.h"
#include "storage/multi_data_source/runtime_utility/mds_factory.h"
#include "mtlenv/storage/medium_info_common.h"
#include "storage/tablet/ob_tablet.h"
using namespace oceanbase::common;
using namespace oceanbase::share;
using namespace oceanbase::unittest;
#define USING_LOG_PREFIX STORAGE
#define MOCK_INSERT_MDS_TABLE(id) \
{ \
mds::MdsCtx ctx##id(mds::MdsWriter(transaction::ObTransID(id))); \
compaction::ObMediumCompactionInfoKey key##id(id); \
compaction::ObMediumCompactionInfo info##id; \
ret = MediumInfoHelper::build_medium_compaction_info(allocator_, info##id, id); \
ASSERT_EQ(OB_SUCCESS, ret); \
\
ret = mds_table_.set(key##id, info##id, ctx##id); \
ASSERT_EQ(OB_SUCCESS, ret); \
\
ctx##id.on_redo(mock_scn(id)); \
ctx##id.before_prepare(); \
ctx##id.on_prepare(mock_scn(id)); \
ctx##id.on_commit(mock_scn(id), mock_scn(id)); \
}
#define MOCK_INSERT_TABLET(id) \
{ \
compaction::ObMediumCompactionInfoKey key##id(id); \
compaction::ObMediumCompactionInfo info##id; \
MediumInfoHelper::build_medium_compaction_info(allocator_, info##id, id); \
mds::MdsDumpKey dump_key##id; \
ret = convert(key##id, dump_key##id); \
ASSERT_EQ(OB_SUCCESS, ret); \
mds::MdsDumpNode dump_node##id; \
ret = convert(info##id, dump_node##id); \
ASSERT_EQ(OB_SUCCESS, ret); \
\
ret = medium_info_list.append(dump_key##id, dump_node##id); \
ASSERT_EQ(OB_SUCCESS, ret); \
}
namespace oceanbase
{
namespace storage
{
class TestMediumInfoReader : public ::testing::Test
class TestMediumInfoReader : public MediumInfoCommon
{
public:
TestMediumInfoReader();
TestMediumInfoReader() = default;
virtual ~TestMediumInfoReader() = default;
public:
virtual void SetUp() override;
virtual void TearDown() override;
static void SetUpTestCase();
static void TearDownTestCase();
public:
static int create_ls(const uint64_t tenant_id, const share::ObLSID &ls_id, ObLSHandle &ls_handle);
static int remove_ls(const share::ObLSID &ls_id);
int create_tablet(const common::ObTabletID &tablet_id, ObTabletHandle &tablet_handle);
public:
int convert(const compaction::ObMediumCompactionInfoKey &key, mds::MdsDumpKey &dump_key);
int convert(const compaction::ObMediumCompactionInfo &info, mds::MdsDumpNode &dump_node);
public:
static constexpr uint64_t TENANT_ID = 1001;
static const share::ObLSID LS_ID;
mds::MdsTableHandle mds_table_;
common::ObArenaAllocator allocator_; // for medium info
};
const share::ObLSID TestMediumInfoReader::LS_ID(1001);
TestMediumInfoReader::TestMediumInfoReader()
: mds_table_(),
allocator_()
{
}
void TestMediumInfoReader::SetUp()
{
}
void TestMediumInfoReader::TearDown()
{
}
void TestMediumInfoReader::SetUpTestCase()
{
int ret = OB_SUCCESS;
ret = MockTenantModuleEnv::get_instance().init();
ASSERT_EQ(OB_SUCCESS, ret);
ObServerCheckpointSlogHandler::get_instance().is_started_ = true;
// create ls
ObLSHandle ls_handle;
ret = create_ls(TENANT_ID, LS_ID, ls_handle);
ASSERT_EQ(OB_SUCCESS, ret);
}
void TestMediumInfoReader::TearDownTestCase()
TEST_F(TestMediumInfoReader, read_multi_medium_info_from_minor)
{
int ret = OB_SUCCESS;
// remove ls
ret = remove_ls(LS_ID);
ASSERT_EQ(OB_SUCCESS, ret);
MockTenantModuleEnv::get_instance().destroy();
}
int TestMediumInfoReader::create_ls(const uint64_t tenant_id, const share::ObLSID &ls_id, ObLSHandle &ls_handle)
{
int ret = OB_SUCCESS;
ret = TestDmlCommon::create_ls(tenant_id, ls_id, ls_handle);
return ret;
}
int TestMediumInfoReader::remove_ls(const share::ObLSID &ls_id)
{
int ret = OB_SUCCESS;
ret = MTL(ObLSService*)->remove_ls(ls_id);
return ret;
}
int TestMediumInfoReader::create_tablet(const common::ObTabletID &tablet_id, ObTabletHandle &tablet_handle)
{
int ret = OB_SUCCESS;
const uint64_t table_id = 1234567;
share::schema::ObTableSchema table_schema;
// get ls
ObLSHandle ls_handle;
ObLS *ls = nullptr;
mds_table_.reset();
ret = MTL(ObLSService*)->get_ls(LS_ID, ls_handle, ObLSGetMod::STORAGE_MOD);
ASSERT_EQ(OB_SUCCESS, ret);
ls = ls_handle.get_ls();
ASSERT_NE(nullptr, ls);
if (OB_FAIL(MTL(ObLSService*)->get_ls(LS_ID, ls_handle, ObLSGetMod::STORAGE_MOD))) {
LOG_WARN("failed to get ls", K(ret));
} else if (OB_ISNULL(ls = ls_handle.get_ls())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls is null", K(ret), KP(ls));
} else if (OB_FAIL(build_test_schema(table_schema, table_id))) {
LOG_WARN("failed to build table schema");
} else if (OB_FAIL(TestTabletHelper::create_tablet(ls_handle, tablet_id, table_schema, allocator_))) {
LOG_WARN("failed to create tablet", K(ret));
} else if (OB_FAIL(ls->get_tablet(tablet_id, tablet_handle))) {
LOG_WARN("failed to get tablet", K(ret));
} else if (OB_FAIL(tablet_handle.get_obj()->inner_get_mds_table(mds_table_, true/*not_exist_create*/))) {
LOG_WARN("failed to get mds table", K(ret));
// create tablet
const common::ObTabletID tablet_id(ObTimeUtility::fast_current_time() % 10000000000000);
ObTabletHandle tablet_handle;
ret = create_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *tablet = tablet_handle.get_obj();
{
ObTabletCreateDeleteMdsUserData user_data;
user_data.tablet_status_ = ObTabletStatus::NORMAL;
user_data.data_type_ = ObTabletMdsUserDataType::CREATE_TABLET;
share::SCN commit_scn = share::SCN::plus(share::SCN::min_scn(), 5);
user_data.create_commit_scn_ = commit_scn;
user_data.create_commit_version_ = 5;
mds::MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(5)));
ret = tablet->set_tablet_status(user_data, ctx);
ASSERT_EQ(OB_SUCCESS, ret);
ctx.single_log_commit(commit_scn, commit_scn);
}
return ret;
}
{
// insert data into mds table
ret = insert_medium_info(1718647202742212010, 1718647202742212010);
ASSERT_EQ(OB_SUCCESS, ret);
int TestMediumInfoReader::convert(const compaction::ObMediumCompactionInfoKey &key, mds::MdsDumpKey &dump_key)
{
int ret = OB_SUCCESS;
constexpr uint8_t table_id = mds::TupleTypeIdx<mds::MdsTableTypeTuple, mds::NormalMdsTable>::value;
constexpr uint8_t unit_id = mds::TupleTypeIdx<mds::NormalMdsTable, mds::MdsUnit<compaction::ObMediumCompactionInfoKey, compaction::ObMediumCompactionInfo>>::value;
// mds table flush
share::SCN decided_scn = share::SCN::plus(share::SCN::min_scn(), 1718647202742212015);
ret = tablet->mds_table_flush(decided_scn);
ASSERT_EQ(OB_SUCCESS, ret);
dump_key.mds_table_id_ = table_id;
dump_key.mds_unit_id_ = unit_id;
dump_key.crc_check_number_ = 0;
dump_key.allocator_ = &allocator_;
// wait for mds table flush
tablet_handle.reset();
ret = wait_for_mds_table_flush(tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
const int64_t length = key.get_serialize_size();
if (0 == length) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("key length should not be 0", K(ret), K(length));
} else {
char *buffer = static_cast<char*>(allocator_.alloc(length));
int64_t pos = 0;
if (OB_ISNULL(buffer)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to alloc memory", K(ret), K(length));
} else if (OB_FAIL(key.serialize(buffer, length, pos))) {
LOG_WARN("failed to serialize", K(ret));
} else {
dump_key.key_.assign(buffer, length);
}
// wait all mds nodes to be released
ret = wait_for_all_mds_nodes_released(tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
}
if (OB_FAIL(ret)) {
if (nullptr != buffer) {
allocator_.free(buffer);
{
// insert data into mds table
ret = insert_medium_info(1718647202742212020, 1718647202742212020);
ASSERT_EQ(OB_SUCCESS, ret);
ret = insert_medium_info(1718647202742212030, 1718647202742212030);
ASSERT_EQ(OB_SUCCESS, ret);
ObTabletHandle tablet_handle;
ret = MediumInfoCommon::get_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
tablet = tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
// mds table flush
share::SCN decided_scn = share::SCN::plus(share::SCN::min_scn(), 1718647202742212035);
ret = tablet->mds_table_flush(decided_scn);
ASSERT_EQ(OB_SUCCESS, ret);
}
// check mds minor merge has done
{
int times = 0;
bool has_minor = false;
do {
ObTabletHandle tablet_handle;
ObTablet *tablet = nullptr;
ret = MediumInfoCommon::get_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
tablet = tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
ObTabletMemberWrapper<ObTabletTableStore> table_store_wrapper;
ret = tablet->fetch_table_store(table_store_wrapper);
ASSERT_EQ(OB_SUCCESS, ret);
const ObTabletTableStore *table_store = table_store_wrapper.get_member();
if (table_store->mds_sstables_.count() != 1) {
// sleep
::ob_usleep(100_ms);
++times;
continue;
} else {
ObSSTable *sstable = table_store->mds_sstables_[0];
has_minor = sstable->is_mds_minor_sstable();
if (!has_minor) {
// sleep
::ob_usleep(100_ms);
++times;
}
}
}
} while (ret == OB_SUCCESS && !has_minor && times < 60);
}
return ret;
}
int TestMediumInfoReader::convert(const compaction::ObMediumCompactionInfo &info, mds::MdsDumpNode &dump_node)
{
int ret = OB_SUCCESS;
constexpr uint8_t table_id = mds::TupleTypeIdx<mds::MdsTableTypeTuple, mds::NormalMdsTable>::value;
constexpr uint8_t unit_id = mds::TupleTypeIdx<mds::NormalMdsTable, mds::MdsUnit<compaction::ObMediumCompactionInfoKey, compaction::ObMediumCompactionInfo>>::value;
dump_node.mds_table_id_ = table_id;
dump_node.mds_unit_id_ = unit_id;
dump_node.crc_check_number_ = 0;
dump_node.status_.union_.field_.node_type_ = mds::MdsNodeType::SET;
dump_node.status_.union_.field_.writer_type_ = mds::WriterType::TRANSACTION;
dump_node.status_.union_.field_.state_ = mds::TwoPhaseCommitState::ON_COMMIT;
dump_node.allocator_ = &allocator_;
dump_node.writer_id_ = 1;
dump_node.seq_no_ = 1;
dump_node.redo_scn_ = share::SCN::max_scn();
dump_node.end_scn_ = share::SCN::max_scn();
dump_node.trans_version_ = share::SCN::max_scn();
const int64_t length = info.get_serialize_size();
if (0 == length) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("info length should not be 0", K(ret), K(length));
} else {
char *buffer = static_cast<char*>(allocator_.alloc(length));
int64_t pos = 0;
if (OB_ISNULL(buffer)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to alloc memory", K(ret), K(length));
} else if (OB_FAIL(info.serialize(buffer, length, pos))) {
LOG_WARN("failed to serialize", K(ret));
} else {
dump_node.user_data_.assign(buffer, length);
}
if (OB_FAIL(ret)) {
if (nullptr != buffer) {
allocator_.free(buffer);
}
}
}
return ret;
}
TEST_F(TestMediumInfoReader, pure_mds_table)
{
int ret = OB_SUCCESS;
// create tablet
const common::ObTabletID tablet_id(ObTimeUtility::fast_current_time() % 10000000000000);
ObTabletHandle tablet_handle;
ret = create_tablet(tablet_id, tablet_handle);
// read medium info
ret = MediumInfoCommon::get_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *tablet = tablet_handle.get_obj();
tablet = tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
// insert data into mds table
MOCK_INSERT_MDS_TABLE(1);
MOCK_INSERT_MDS_TABLE(2);
// iterate
{
common::ObArenaAllocator allocator;
ObTabletMediumInfoReader reader(*tablet);
ret = reader.init(allocator);
ASSERT_EQ(OB_SUCCESS, ret);
compaction::ObMediumCompactionInfoKey key;
compaction::ObMediumCompactionInfo info;
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(1, key.medium_snapshot_);
ASSERT_EQ(1, info.medium_snapshot_);
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(2, key.medium_snapshot_);
ASSERT_EQ(2, info.medium_snapshot_);
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_ITER_END, ret);
reader.reset();
}
{
common::ObArenaAllocator allocator;
ObTabletMediumInfoReader reader(*tablet);
ASSERT_EQ(OB_SUCCESS, reader.init(allocator));
int64_t medium_snapshot = 0;
ret = reader.get_max_medium_snapshot(medium_snapshot);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(medium_snapshot, 2);
}
{
common::ObArenaAllocator allocator;
ObTabletMediumInfoReader reader(*tablet);
ASSERT_EQ(OB_SUCCESS, reader.init(allocator));
int64_t medium_snapshot = 0;
ret = reader.get_min_medium_snapshot(0, medium_snapshot);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(medium_snapshot, 1);
ret = reader.get_min_medium_snapshot(1, medium_snapshot);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(medium_snapshot, 2);
}
}
TEST_F(TestMediumInfoReader, pure_dump_data)
{
int ret = OB_SUCCESS;
// create tablet
const common::ObTabletID tablet_id(ObTimeUtility::fast_current_time() % 10000000000000);
ObTabletHandle tablet_handle;
ObTabletHandle new_tablet_handle;
ret = create_tablet(tablet_id, tablet_handle);
common::ObSEArray<compaction::ObMediumCompactionInfo*, 1> medium_info_array;
ret = tablet->read_medium_array(allocator_, medium_info_array);
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *tablet = tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
ObTabletPointer *tablet_ptr = static_cast<ObTabletPointer*>(tablet->get_pointer_handle().get_resource_ptr());
ret = tablet_ptr->try_gc_mds_table();
ASSERT_EQ(OB_SUCCESS, ret);
// insert data into tablet
{
common::ObIAllocator &allocator = *tablet_handle.get_allocator();
ObTabletComplexAddr<ObTabletDumpedMediumInfo> &medium_info_list_complex_addr = tablet->mds_data_.medium_info_list_;
ret = ObTabletObjLoadHelper::alloc_and_new(allocator, medium_info_list_complex_addr.ptr_);
ASSERT_EQ(OB_SUCCESS, ret);
ret = medium_info_list_complex_addr.ptr_->init_for_first_creation(allocator);
ASSERT_EQ(OB_SUCCESS, ret);
ObTabletDumpedMediumInfo &medium_info_list = *medium_info_list_complex_addr.ptr_;
constexpr uint8_t table_id = mds::TupleTypeIdx<mds::MdsTableTypeTuple, mds::NormalMdsTable>::value;
constexpr uint8_t unit_id = mds::TupleTypeIdx<mds::NormalMdsTable, mds::MdsUnit<compaction::ObMediumCompactionInfoKey, compaction::ObMediumCompactionInfo>>::value;
MOCK_INSERT_TABLET(1);
MOCK_INSERT_TABLET(2);
// persist
ret = ObTabletPersister::persist_and_transform_tablet(*tablet, new_tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
tablet = new_tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
}
// iterate
{
common::ObArenaAllocator allocator;
ObTabletMediumInfoReader reader(*tablet);
ret = reader.init(allocator);
ASSERT_EQ(OB_SUCCESS, ret);
compaction::ObMediumCompactionInfoKey key;
compaction::ObMediumCompactionInfo info;
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(1, key.medium_snapshot_);
ASSERT_EQ(1, info.medium_snapshot_);
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(2, key.medium_snapshot_);
ASSERT_EQ(2, info.medium_snapshot_);
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_ITER_END, ret);
}
}
TEST_F(TestMediumInfoReader, mds_table_dump_data_overlap)
{
int ret = OB_SUCCESS;
// create tablet
const common::ObTabletID tablet_id(ObTimeUtility::fast_current_time() % 10000000000000);
ObTabletHandle tablet_handle;
ObTabletHandle new_tablet_handle;
ret = create_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *tablet = tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
// insert data into mds table
MOCK_INSERT_MDS_TABLE(2);
MOCK_INSERT_MDS_TABLE(3);
// insert data into mds data
{
common::ObIAllocator &allocator = *tablet_handle.get_allocator();
ObTabletComplexAddr<ObTabletDumpedMediumInfo> &medium_info_list_complex_addr = tablet->mds_data_.medium_info_list_;
ret = ObTabletObjLoadHelper::alloc_and_new(allocator, medium_info_list_complex_addr.ptr_);
ASSERT_EQ(OB_SUCCESS, ret);
ret = medium_info_list_complex_addr.ptr_->init_for_first_creation(allocator);
ASSERT_EQ(OB_SUCCESS, ret);
ObTabletDumpedMediumInfo &medium_info_list = *medium_info_list_complex_addr.ptr_;
constexpr uint8_t table_id = mds::TupleTypeIdx<mds::MdsTableTypeTuple, mds::NormalMdsTable>::value;
constexpr uint8_t unit_id = mds::TupleTypeIdx<mds::NormalMdsTable, mds::MdsUnit<compaction::ObMediumCompactionInfoKey, compaction::ObMediumCompactionInfo>>::value;
MOCK_INSERT_TABLET(1);
MOCK_INSERT_TABLET(2);
// persist
ret = ObTabletPersister::persist_and_transform_tablet(*tablet, new_tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
tablet = new_tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
}
// iterate
{
common::ObArenaAllocator allocator;
ObTabletMediumInfoReader reader(*tablet);
ret = reader.init(allocator);
ASSERT_EQ(OB_SUCCESS, ret);
compaction::ObMediumCompactionInfoKey key;
compaction::ObMediumCompactionInfo info;
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(1, key.medium_snapshot_);
ASSERT_EQ(1, info.medium_snapshot_);
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(2, key.medium_snapshot_);
ASSERT_EQ(2, info.medium_snapshot_);
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(3, key.medium_snapshot_);
ASSERT_EQ(3, info.medium_snapshot_);
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_ITER_END, ret);
}
{
common::ObArenaAllocator allocator;
ObTabletMediumInfoReader reader(*tablet);
ASSERT_EQ(OB_SUCCESS, reader.init(allocator));
int64_t medium_snapshot = 0;
ret = reader.get_max_medium_snapshot(medium_snapshot);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(medium_snapshot, 3);
}
{
common::ObArenaAllocator allocator;
ObTabletMediumInfoReader reader(*tablet);
ASSERT_EQ(OB_SUCCESS, reader.init(allocator));
int64_t medium_snapshot = 0;
ret = reader.get_min_medium_snapshot(0, medium_snapshot);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(medium_snapshot, 1);
}
{
common::ObArenaAllocator allocator;
ObTabletMediumInfoReader reader(*tablet);
ASSERT_EQ(OB_SUCCESS, reader.init(allocator));
ObArenaAllocator tmp_allocator;
compaction::ObMediumCompactionInfoKey key(1);
compaction::ObMediumCompactionInfo medium_info;
ret = reader.get_specified_medium_info(tmp_allocator, key, medium_info);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(medium_info.medium_snapshot_, 1);
}
{
common::ObArenaAllocator allocator;
ObTabletMediumInfoReader reader(*tablet);
ASSERT_EQ(OB_SUCCESS, reader.init(allocator));
ObArenaAllocator tmp_allocator;
compaction::ObMediumCompactionInfoKey key(10);
compaction::ObMediumCompactionInfo medium_info;
ret = reader.get_specified_medium_info(tmp_allocator, key, medium_info);
ASSERT_EQ(OB_ENTRY_NOT_EXIST, ret);
}
}
TEST_F(TestMediumInfoReader, mds_table_dump_data_no_overlap)
{
int ret = OB_SUCCESS;
// create tablet
const common::ObTabletID tablet_id(ObTimeUtility::fast_current_time() % 10000000000000);
ObTabletHandle tablet_handle;
ObTabletHandle new_tablet_handle;
ret = create_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *tablet = tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
// insert data into mds table
MOCK_INSERT_MDS_TABLE(5);
MOCK_INSERT_MDS_TABLE(8);
// insert data into mds data
{
common::ObIAllocator &allocator = *tablet_handle.get_allocator();
ObTabletComplexAddr<ObTabletDumpedMediumInfo> &medium_info_list_complex_addr = tablet->mds_data_.medium_info_list_;
ret = ObTabletObjLoadHelper::alloc_and_new(allocator, medium_info_list_complex_addr.ptr_);
ASSERT_EQ(OB_SUCCESS, ret);
ret = medium_info_list_complex_addr.ptr_->init_for_first_creation(allocator);
ASSERT_EQ(OB_SUCCESS, ret);
ObTabletDumpedMediumInfo &medium_info_list = *medium_info_list_complex_addr.ptr_;
constexpr uint8_t table_id = mds::TupleTypeIdx<mds::MdsTableTypeTuple, mds::NormalMdsTable>::value;
constexpr uint8_t unit_id = mds::TupleTypeIdx<mds::NormalMdsTable, mds::MdsUnit<compaction::ObMediumCompactionInfoKey, compaction::ObMediumCompactionInfo>>::value;
MOCK_INSERT_TABLET(1);
MOCK_INSERT_TABLET(2);
// persist
ret = ObTabletPersister::persist_and_transform_tablet(*tablet, new_tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
tablet = new_tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
}
// iterate
{
common::ObArenaAllocator allocator;
ObTabletMediumInfoReader reader(*tablet);
ret = reader.init(allocator);
ASSERT_EQ(OB_SUCCESS, ret);
compaction::ObMediumCompactionInfoKey key;
compaction::ObMediumCompactionInfo info;
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(1, key.medium_snapshot_);
ASSERT_EQ(1, info.medium_snapshot_);
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(2, key.medium_snapshot_);
ASSERT_EQ(2, info.medium_snapshot_);
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(5, key.medium_snapshot_);
ASSERT_EQ(5, info.medium_snapshot_);
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(8, key.medium_snapshot_);
ASSERT_EQ(8, info.medium_snapshot_);
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_ITER_END, ret);
reader.reset();
}
{
common::ObArenaAllocator allocator;
ObTabletMediumInfoReader reader(*tablet);
ASSERT_EQ(OB_SUCCESS, reader.init(allocator));
int64_t medium_snapshot = 0;
ret = reader.get_max_medium_snapshot(medium_snapshot);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(medium_snapshot, 8);
}
{
common::ObArenaAllocator allocator;
ObTabletMediumInfoReader reader(*tablet);
ASSERT_EQ(OB_SUCCESS, reader.init(allocator));
int64_t medium_snapshot = 0;
ret = reader.get_min_medium_snapshot(0, medium_snapshot);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(medium_snapshot, 1);
}
}
TEST_F(TestMediumInfoReader, mds_table_dump_data_full_inclusion)
{
int ret = OB_SUCCESS;
// create tablet
const common::ObTabletID tablet_id(ObTimeUtility::fast_current_time() % 10000000000000);
ObTabletHandle tablet_handle;
ObTabletHandle new_tablet_handle;
ret = create_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *tablet = tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
// insert data into mds table
MOCK_INSERT_MDS_TABLE(1);
MOCK_INSERT_MDS_TABLE(2);
MOCK_INSERT_MDS_TABLE(3);
MOCK_INSERT_MDS_TABLE(4);
MOCK_INSERT_MDS_TABLE(5);
// insert data into mds data
{
common::ObIAllocator &allocator = *tablet_handle.get_allocator();
ObTabletComplexAddr<ObTabletDumpedMediumInfo> &medium_info_list_complex_addr = tablet->mds_data_.medium_info_list_;
ret = ObTabletObjLoadHelper::alloc_and_new(allocator, medium_info_list_complex_addr.ptr_);
ASSERT_EQ(OB_SUCCESS, ret);
ret = medium_info_list_complex_addr.ptr_->init_for_first_creation(allocator);
ASSERT_EQ(OB_SUCCESS, ret);
ObTabletDumpedMediumInfo &medium_info_list = *medium_info_list_complex_addr.ptr_;
constexpr uint8_t table_id = mds::TupleTypeIdx<mds::MdsTableTypeTuple, mds::NormalMdsTable>::value;
constexpr uint8_t unit_id = mds::TupleTypeIdx<mds::NormalMdsTable, mds::MdsUnit<compaction::ObMediumCompactionInfoKey, compaction::ObMediumCompactionInfo>>::value;
MOCK_INSERT_TABLET(2);
MOCK_INSERT_TABLET(4);
// persist
ret = ObTabletPersister::persist_and_transform_tablet(*tablet, new_tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
tablet = new_tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
}
// iterate
{
common::ObArenaAllocator allocator;
ObTabletMediumInfoReader reader(*tablet);
ret = reader.init(allocator);
ASSERT_EQ(OB_SUCCESS, ret);
compaction::ObMediumCompactionInfoKey key;
compaction::ObMediumCompactionInfo info;
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(1, key.medium_snapshot_);
ASSERT_EQ(1, info.medium_snapshot_);
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(2, key.medium_snapshot_);
ASSERT_EQ(2, info.medium_snapshot_);
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(3, key.medium_snapshot_);
ASSERT_EQ(3, info.medium_snapshot_);
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(4, key.medium_snapshot_);
ASSERT_EQ(4, info.medium_snapshot_);
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(5, key.medium_snapshot_);
ASSERT_EQ(5, info.medium_snapshot_);
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_ITER_END, ret);
}
}
TEST_F(TestMediumInfoReader, mds_table_cannot_read_uncommitted_node) {
int ret = OB_SUCCESS;
// create tablet
const common::ObTabletID tablet_id(ObTimeUtility::fast_current_time() % 10000000000000);
ObTabletHandle tablet_handle;
ObTabletHandle new_tablet_handle;
ret = create_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *tablet = tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
// insert data into mds table
mds::MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(1)));
compaction::ObMediumCompactionInfoKey key(1);
compaction::ObMediumCompactionInfo info;
ret = MediumInfoHelper::build_medium_compaction_info(allocator_, info, 1);
ASSERT_EQ(OB_SUCCESS, ret);
ret = mds_table_.set(key, info, ctx);
ASSERT_EQ(OB_SUCCESS, ret);
// iterate
{
common::ObArenaAllocator allocator;
ObTabletMediumInfoReader reader(*tablet);
ret = reader.init(allocator);
ASSERT_EQ(OB_SUCCESS, ret);
key.reset();
info.reset();
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_ITER_END, ret);
}
// commit
{
ctx.on_redo(mock_scn(1));
ctx.before_prepare();
ctx.on_prepare(mock_scn(1));
ctx.on_commit(mock_scn(1), mock_scn(1));
}
// iterate
{
common::ObArenaAllocator allocator;
ObTabletMediumInfoReader reader(*tablet);
ret = reader.init(allocator);
ASSERT_EQ(OB_SUCCESS, ret);
ret = reader.get_next_medium_info(allocator, key, info);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(1, key.medium_snapshot_);
ASSERT_EQ(1, info.medium_snapshot_);
}
ASSERT_EQ(3, medium_info_array.count());
ASSERT_EQ(1718647202742212010, medium_info_array[0]->medium_snapshot_);
ASSERT_EQ(1718647202742212020, medium_info_array[1]->medium_snapshot_);
ASSERT_EQ(1718647202742212030, medium_info_array[2]->medium_snapshot_);
}
} // namespace storage
} // namespace oceanbase

404
mittest/mtlenv/storage/test_migration_tablet_param.cpp Normal file
View File

@ -0,0 +1,404 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#define private public
#define protected public
#include "lib/ob_errno.h"
#include "lib/allocator/page_arena.h"
#include "lib/oblog/ob_log.h"
#include "share/rc/ob_tenant_base.h"
#include "mtlenv/mock_tenant_module_env.h"
#include "mtlenv/storage/medium_info_helper.h"
#include "unittest/storage/test_tablet_helper.h"
#include "unittest/storage/test_dml_common.h"
#include "storage/tablet/ob_tablet.h"
#define USING_LOG_PREFIX STORAGE
using namespace oceanbase::common;
using namespace oceanbase::share;
using namespace oceanbase::storage;
namespace oceanbase
{
int ObClusterVersion::get_tenant_data_version(const uint64_t tenant_id, uint64_t &data_version)
{
data_version = DATA_VERSION_4_3_2_0;
return OB_SUCCESS;
}
namespace unittest
{
class TestMigrationTabletParam : public ::testing::Test
{
public:
TestMigrationTabletParam() = default;
virtual ~TestMigrationTabletParam() = default;
public:
static constexpr uint64_t TENANT_ID = 1001;
static const share::ObLSID LS_ID;
public:
static void SetUpTestCase();
static void TearDownTestCase();
public:
static int create_ls(const uint64_t tenant_id, const share::ObLSID &ls_id, ObLSHandle &ls_handle);
static int remove_ls(const share::ObLSID &ls_id);
int create_tablet(const common::ObTabletID &tablet_id, ObTabletHandle &tablet_handle);
static int insert_data_into_mds_data(ObArenaAllocator &allocator, mds::MdsDumpKV &kv);
static bool is_user_data_equal(const ObTabletCreateDeleteMdsUserData &left, const ObTabletCreateDeleteMdsUserData &right);
public:
ObArenaAllocator allocator_;
};
const share::ObLSID TestMigrationTabletParam::LS_ID(1234);
void TestMigrationTabletParam::SetUpTestCase()
{
int ret = OB_SUCCESS;
ret = MockTenantModuleEnv::get_instance().init();
ASSERT_EQ(OB_SUCCESS, ret);
ObServerCheckpointSlogHandler::get_instance().is_started_ = true;
// create ls
ObLSHandle ls_handle;
ret = create_ls(TENANT_ID, LS_ID, ls_handle);
ASSERT_EQ(OB_SUCCESS, ret);
}
void TestMigrationTabletParam::TearDownTestCase()
{
int ret = OB_SUCCESS;
// remove ls
ret = remove_ls(LS_ID);
ASSERT_EQ(OB_SUCCESS, ret);
MockTenantModuleEnv::get_instance().destroy();
}
int TestMigrationTabletParam::create_ls(const uint64_t tenant_id, const share::ObLSID &ls_id, ObLSHandle &ls_handle)
{
int ret = OB_SUCCESS;
ret = TestDmlCommon::create_ls(tenant_id, ls_id, ls_handle);
return ret;
}
int TestMigrationTabletParam::remove_ls(const share::ObLSID &ls_id)
{
int ret = OB_SUCCESS;
ret = MTL(ObLSService*)->remove_ls(ls_id);
return ret;
}
int TestMigrationTabletParam::create_tablet(const common::ObTabletID &tablet_id, ObTabletHandle &tablet_handle)
{
int ret = OB_SUCCESS;
const uint64_t table_id = 1234567;
share::schema::ObTableSchema table_schema;
ObLSHandle ls_handle;
ObLS *ls = nullptr;
if (OB_FAIL(MTL(ObLSService*)->get_ls(LS_ID, ls_handle, ObLSGetMod::STORAGE_MOD))) {
LOG_WARN("failed to get ls", K(ret));
} else if (OB_ISNULL(ls = ls_handle.get_ls())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls is null", K(ret), KP(ls));
} else if (OB_FAIL(build_test_schema(table_schema, table_id))) {
LOG_WARN("failed to build table schema");
} else if (OB_FAIL(TestTabletHelper::create_tablet(ls_handle, tablet_id, table_schema, allocator_,
ObTabletStatus::MAX, share::SCN::invalid_scn(), tablet_handle))) {
LOG_WARN("failed to create tablet", K(ret));
}
return ret;
}
int TestMigrationTabletParam::insert_data_into_mds_data(ObArenaAllocator &allocator, mds::MdsDumpKV &kv)
{
int ret = OB_SUCCESS;
ObTabletCreateDeleteMdsUserData user_data;
user_data.tablet_status_ = ObTabletStatus::NORMAL;
const int64_t serialize_size = user_data.get_serialize_size();
char *buffer = static_cast<char *>(allocator.alloc(serialize_size));
int64_t pos = 0;
if (OB_ISNULL(buffer)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to alloc memory", K(ret), K(serialize_size));
} else if (OB_FAIL(user_data.serialize(buffer, serialize_size, pos))) {
LOG_WARN("fail to serialize user data", K(ret));
} else {
kv.v_.user_data_.assign(buffer, serialize_size);
}
return ret;
}
bool TestMigrationTabletParam::is_user_data_equal(const ObTabletCreateDeleteMdsUserData &left, const ObTabletCreateDeleteMdsUserData &right)
{
const bool res = left.tablet_status_ == right.tablet_status_ && left.transfer_scn_ == right.transfer_scn_
&& left.transfer_ls_id_ == right.transfer_ls_id_ && left.data_type_ == right.data_type_
&& left.create_commit_scn_ == right.create_commit_scn_
&& left.create_commit_version_ == right.create_commit_version_
&& left.delete_commit_version_ == right.delete_commit_version_
&& left.transfer_out_commit_version_ == right.transfer_out_commit_version_;
return res;
}
TEST_F(TestMigrationTabletParam, migration)
{
int ret = OB_SUCCESS;
// create tablet
const common::ObTabletID tablet_id(ObTimeUtility::fast_current_time() % 10000000000000);
ObTabletHandle tablet_handle;
ObTabletStatus status(ObTabletStatus::MAX);
share::SCN create_commit_scn;
create_commit_scn = share::SCN::plus(share::SCN::min_scn(), 100);
ret = create_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *tablet = tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
// write data to mds table
{
ObTabletCreateDeleteMdsUserData user_data;
user_data.tablet_status_ = ObTabletStatus::NORMAL;
user_data.data_type_ = ObTabletMdsUserDataType::CREATE_TABLET;
mds::MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(123)));
ret = tablet->set_tablet_status(user_data, ctx);
ASSERT_EQ(OB_SUCCESS, ret);
share::SCN redo_scn = share::SCN::plus(share::SCN::min_scn(), 50);
ctx.on_redo(redo_scn);
ctx.on_commit(create_commit_scn, create_commit_scn);
}
// {
// // build migration param v2
// ObMigrationTabletParam param_v2;
// ret = tablet->build_migration_tablet_param(param_v2);
// ASSERT_EQ(OB_NOT_SUPPORTED, ret);
// // insert some data into migration param
// ret = insert_data_into_mds_data(allocator_, param_v2.mds_data_.tablet_status_committed_kv_);
// ASSERT_EQ(OB_SUCCESS, ret);
// param_v2.version_ = ObMigrationTabletParam::PARAM_VERSION_V2;
// int64_t serialize_size = param_v2.get_serialize_size();
// char *buffer = new char[serialize_size]();
// int64_t pos = 0;
// ret = param_v2.serialize(buffer, serialize_size, pos);
// ASSERT_EQ(OB_SUCCESS, ret);
// ASSERT_EQ(serialize_size, pos);
// // param v2 deserialize
// ObMigrationTabletParam param2;
// pos = 0;
// ret = param2.deserialize(buffer, serialize_size, pos);
// ASSERT_EQ(OB_SUCCESS, ret);
// ASSERT_EQ(pos, serialize_size);
// ASSERT_TRUE(param2.is_valid());
// delete [] buffer;
// }
{
// build migration param v3
ObMigrationTabletParam param_v3;
ret = tablet->build_migration_tablet_param(param_v3);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(true, param_v3.version_ == ObMigrationTabletParam::PARAM_VERSION_V3);
// insert some data into migration param
ret = insert_data_into_mds_data(allocator_, param_v3.mds_data_.tablet_status_committed_kv_);
ASSERT_EQ(OB_SUCCESS, ret);
int64_t serialize_size = param_v3.get_serialize_size();
char *buffer = new char[serialize_size]();
int64_t pos = 0;
ret = param_v3.serialize(buffer, serialize_size, pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(serialize_size, pos);
// param v3 deserialize
ObMigrationTabletParam param2;
pos = 0;
ret = param2.deserialize(buffer, serialize_size, pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(pos, serialize_size);
ASSERT_TRUE(param2.is_valid());
delete [] buffer;
}
}
TEST_F(TestMigrationTabletParam, transfer)
{
int ret = OB_SUCCESS;
// create tablet
const common::ObTabletID tablet_id(ObTimeUtility::fast_current_time() % 10000000000000);
ObTabletHandle tablet_handle;
ObTabletStatus status(ObTabletStatus::MAX);
ret = create_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *tablet = tablet_handle.get_obj();
ASSERT_NE(nullptr, tablet);
// write data to mds table
{
ObTabletCreateDeleteMdsUserData user_data;
user_data.tablet_status_ = ObTabletStatus::TRANSFER_OUT;
user_data.data_type_ = ObTabletMdsUserDataType::START_TRANSFER_OUT;
user_data.transfer_scn_ = share::SCN::plus(share::SCN::min_scn(), 60);
share::SCN transfer_commit_scn;
transfer_commit_scn = share::SCN::plus(share::SCN::min_scn(), 70);
mds::MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(123)));
ret = tablet->set_tablet_status(user_data, ctx);
ASSERT_EQ(OB_SUCCESS, ret);
share::SCN redo_scn = user_data.transfer_scn_;
ctx.on_redo(redo_scn);
ctx.on_commit(transfer_commit_scn, transfer_commit_scn);
}
// write medium info
{
compaction::ObMediumCompactionInfoKey key(100);
compaction::ObMediumCompactionInfo info;
ret = MediumInfoHelper::build_medium_compaction_info(allocator_, info, 100);
ASSERT_EQ(OB_SUCCESS, ret);
mds::MdsCtx ctx(mds::MdsWriter(transaction::ObTransID(777)));
ret = tablet->set(key, info, ctx, 1_s/*lock_timeout_us*/);
ASSERT_EQ(OB_SUCCESS, ret);
share::SCN redo_scn = share::SCN::plus(share::SCN::min_scn(), 110);
ctx.on_redo(redo_scn);
share::SCN commit_scn = share::SCN::plus(share::SCN::min_scn(), 120);
ctx.on_commit(commit_scn, commit_scn);
}
{
// build transfer tablet param v3
ObMigrationTabletParam param_v3;
ASSERT_EQ(false, param_v3.is_valid());
share::ObLSID dst_ls_id(1001);
const int64_t data_version = DATA_VERSION_4_3_2_0;
ret = tablet->build_transfer_tablet_param(data_version, dst_ls_id, param_v3);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(true, param_v3.is_valid());
ASSERT_EQ(true, param_v3.version_ == ObMigrationTabletParam::PARAM_VERSION_V3);
ASSERT_EQ(param_v3.last_persisted_committed_tablet_status_.data_type_, ObTabletMdsUserDataType::START_TRANSFER_IN);
ASSERT_EQ(param_v3.last_persisted_committed_tablet_status_.tablet_status_, ObTabletStatus::TRANSFER_IN);
int64_t serialize_size = param_v3.get_serialize_size();
char *buffer = new char[serialize_size]();
int64_t pos = 0;
ret = param_v3.serialize(buffer, serialize_size, pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(serialize_size, pos);
// param v3 deserialize
ObMigrationTabletParam param2;
pos = 0;
ret = param2.deserialize(buffer, serialize_size, pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(pos, serialize_size);
ASSERT_TRUE(param2.is_valid());
ASSERT_EQ(true, is_user_data_equal(param_v3.last_persisted_committed_tablet_status_, param2.last_persisted_committed_tablet_status_));
ObMigrationTabletParam test_param;
test_param.assign(param_v3);
ASSERT_EQ(true, is_user_data_equal(param_v3.last_persisted_committed_tablet_status_, test_param.last_persisted_committed_tablet_status_));
delete [] buffer;
}
}
TEST_F(TestMigrationTabletParam, empty_shell_transfer)
{
int ret = OB_SUCCESS;
// create tablet
const common::ObTabletID tablet_id(ObTimeUtility::fast_current_time() % 10000000000000);
share::schema::ObTableSchema schema;
ObLSHandle ls_handle;
ObLS *ls = nullptr;
ObLSTabletService *ls_tablet_service = nullptr;
if (OB_FAIL(MTL(ObLSService*)->get_ls(LS_ID, ls_handle, ObLSGetMod::STORAGE_MOD))) {
LOG_WARN("failed to get ls", K(ret));
} else if (OB_ISNULL(ls = ls_handle.get_ls())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls is null", K(ret), KP(ls));
} else {
ls_tablet_service = ls->get_tablet_svr();
}
ASSERT_NE(nullptr, ls_tablet_service);
TestSchemaUtils::prepare_data_schema(schema);
ret = TestTabletHelper::create_tablet(ls_handle, tablet_id, schema, allocator_, ObTabletStatus::Status::DELETED);
ASSERT_EQ(OB_SUCCESS, ret);
ret = ls_tablet_service->update_tablet_to_empty_shell(tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
ObTabletHandle empty_shell_tablet_handle;
ret = ls_tablet_service->get_tablet(tablet_id, empty_shell_tablet_handle, 0, ObMDSGetTabletMode::READ_WITHOUT_CHECK);
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *empty_shell_tablet = empty_shell_tablet_handle.get_obj();
ASSERT_NE(nullptr, empty_shell_tablet);
{
// build transfer tablet param v3
ObMigrationTabletParam param_v3;
ASSERT_EQ(false, param_v3.is_valid());
share::ObLSID dst_ls_id(1001);
const int64_t data_version = DATA_VERSION_4_3_2_0;
ret = empty_shell_tablet->build_transfer_tablet_param(data_version, dst_ls_id, param_v3);
ASSERT_EQ(OB_ERR_UNEXPECTED, ret);
}
{
// build migration tablet param v3
ObMigrationTabletParam param_v3;
ASSERT_EQ(false, param_v3.is_valid());
ASSERT_EQ(false, param_v3.is_valid());
ret = empty_shell_tablet->build_migration_tablet_param(param_v3);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(true, param_v3.is_valid());
ASSERT_EQ(true, param_v3.version_ == ObMigrationTabletParam::PARAM_VERSION_V3);
ObMigrationTabletParam test_param;
test_param.assign(param_v3);
ASSERT_EQ(true, is_user_data_equal(param_v3.last_persisted_committed_tablet_status_, test_param.last_persisted_committed_tablet_status_));
}
}
} // namespace unittest
} // namespace oceanbase
int main(int argc, char **argv)
{
system("rm -rf test_migration_tablet_param.log*");
OB_LOGGER.set_file_name("test_migration_tablet_param.log", true);
OB_LOGGER.set_log_level("INFO");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

294
mittest/mtlenv/storage/test_tablet_status.cpp
View File

@ -1,294 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#include <thread>
#define USING_LOG_PREFIX STORAGE
#define private public
#define protected public
#include "share/ob_ls_id.h"
#include "storage/tablet/ob_tablet_status.h"
#include "storage/tablet/ob_tablet.h"
#include "storage/tablet/ob_tablet_create_delete_helper.h"
#include "storage/ls/ob_ls.h"
#include "storage/ls/ob_ls_tablet_service.h"
#include "storage/meta_mem/ob_tenant_meta_mem_mgr.h"
#include "mtlenv/mock_tenant_module_env.h"
#include "storage/schema_utils.h"
#include "storage/test_dml_common.h"
#include "share/scn.h"
#include "logservice/palf/log_define.h"
#include "storage/tablet/ob_tablet_table_store_flag.h"
using namespace oceanbase::common;
using namespace oceanbase::share;
namespace oceanbase
{
namespace storage
{
class TestTabletStatus : public ::testing::Test
{
public:
TestTabletStatus(const uint64_t tenant_id = TEST_TENANT_ID);
virtual ~TestTabletStatus() = default;
virtual void SetUp() override;
virtual void TearDown() override;
static void SetUpTestCase();
static void TearDownTestCase();
public:
void wait_for_tablet(
const common::ObTabletID &tablet_id,
ObTabletHandle &tablet_handle);
public:
static const uint64_t TEST_TENANT_ID = 1;
static const uint64_t TEST_LS_ID = 1001;
const uint64_t tenant_id_;
share::ObLSID ls_id_;
ObTenantBase *tenant_base_;
common::ObArenaAllocator allocator_;
};
TestTabletStatus::TestTabletStatus(const uint64_t tenant_id)
: tenant_id_(tenant_id),
ls_id_(TEST_LS_ID),
tenant_base_(nullptr)
{
}
void TestTabletStatus::SetUp()
{
ASSERT_TRUE(MockTenantModuleEnv::get_instance().is_inited());
int ret = OB_SUCCESS;
ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*);
t3m->stop();
t3m->wait();
t3m->destroy();
ret = t3m->init();
ASSERT_EQ(OB_SUCCESS, ret);
tenant_base_ = MTL_CTX();
ASSERT_TRUE(tenant_base_ != nullptr);
}
void TestTabletStatus::TearDown()
{
ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*);
t3m->stop();
t3m->wait();
t3m->destroy();
}
void TestTabletStatus::SetUpTestCase()
{
int ret = OB_SUCCESS;
ret = MockTenantModuleEnv::get_instance().init();
ASSERT_EQ(OB_SUCCESS, ret);
ObServerCheckpointSlogHandler::get_instance().is_started_ = true;
// create ls
ObLSHandle ls_handle;
ret = TestDmlCommon::create_ls(TestSchemaUtils::TEST_TENANT_ID, ObLSID(TEST_LS_ID), ls_handle);
ASSERT_EQ(OB_SUCCESS, ret);
}
void TestTabletStatus::TearDownTestCase()
{
int ret = OB_SUCCESS;
ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*);
t3m->stop();
t3m->wait();
t3m->destroy();
ret = t3m->init();
ASSERT_EQ(OB_SUCCESS, ret);
ret = MTL(ObLSService*)->remove_ls(ObLSID(TEST_LS_ID));
ASSERT_EQ(OB_SUCCESS, ret);
MockTenantModuleEnv::get_instance().destroy();
}
void TestTabletStatus::wait_for_tablet(
const common::ObTabletID &tablet_id,
ObTabletHandle &tablet_handle)
{
int ret = OB_SUCCESS;
const int64_t timeout_us = 3 * 1000 * 1000;
share::ObTenantEnv::set_tenant(tenant_base_);
ObLSHandle ls_handle;
ObLSService *ls_svr = MTL(ObLSService*);
ret = ls_svr->get_ls(ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD);
ASSERT_EQ(OB_SUCCESS, ret);
ObLS *ls = ls_handle.get_ls();
ObLSTabletService &ls_tablet_service = ls->ls_tablet_svr_;
while (true) {
ret = ls_tablet_service.get_tablet(tablet_id, tablet_handle, timeout_us);
if (OB_TIMEOUT == ret) {
LOG_INFO("get tablet timeout", K(ret), K(tablet_id));
continue;
} else if (OB_SUCCESS == ret) {
break;
} else {
LOG_WARN("failed to get tablet", K(ret), K(tablet_id));
break;
}
}
}
TEST_F(TestTabletStatus, misc)
{
int ret = OB_SUCCESS;
ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*);
const ObTabletID tablet_id(123);
const ObTabletMapKey key(ls_id_, tablet_id);
ObTabletHandle tablet_handle;
// get ls
ObLSHandle ls_handle;
ObLSService *ls_svr = MTL(ObLSService*);
ret = ls_svr->get_ls(ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD);
ASSERT_EQ(OB_SUCCESS, ret);
ObLS *ls = ls_handle.get_ls();
ret = t3m->create_msd_tablet(WashTabletPriority::WTP_HIGH, key, ls_handle, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(tablet_handle.is_valid());
// mock inited
ObTableSchema table_schema;
TestSchemaUtils::prepare_data_schema(table_schema);
const transaction::ObTransID tx_id = 1;
const int64_t snapshot_version = 1;
const lib::Worker::CompatMode compat_mode = lib::Worker::CompatMode::MYSQL;
ObTabletID empty_tablet_id;
ObFreezer *freezer = ls->get_freezer();
ObTablet *tablet = tablet_handle.get_obj();
tablet->tablet_meta_.tx_data_.tablet_status_ = ObTabletStatus::CREATING; // mock
ObTabletTableStoreFlag store_flag;
store_flag.set_with_major_sstable();
bool make_empty_co_sstable = false;
ret = tablet->init(allocator_, ls_id_, tablet_id, tablet_id, empty_tablet_id, empty_tablet_id,
share::SCN::base_scn(), snapshot_version, table_schema, compat_mode, store_flag, nullptr, freezer);
ASSERT_EQ(OB_SUCCESS, ret);
ObUpdateTabletPointerParam param;
ret = tablet_handle.get_obj()->get_updating_tablet_pointer_param(param);
ASSERT_EQ(OB_SUCCESS, ret);
ret = t3m->compare_and_swap_tablet(key, tablet_handle, tablet_handle, param);
ASSERT_EQ(OB_SUCCESS, ret);
// set tx data
ObTabletTxMultiSourceDataUnit tx_data;
const bool for_replay = false;
tx_data.tx_id_ = 1;
tx_data.tx_scn_.convert_for_gts(share::OB_MAX_SCN_TS_NS);
tx_data.tablet_status_ = ObTabletStatus::CREATING;
ret = tablet->set_tx_data(tx_data, for_replay);
ASSERT_EQ(OB_SUCCESS, ret);
// get tablet
ObTabletHandle fail_tablet_handle;
ObLSTabletService &ls_tablet_service = ls->ls_tablet_svr_;
const int64_t timeout_us = 100 * 1000;
ret = ls_tablet_service.get_tablet(tablet_id, fail_tablet_handle, timeout_us);
ASSERT_EQ(OB_TIMEOUT, ret);
// mock
ObTabletHandle handle;
std::thread t1(&TestTabletStatus::wait_for_tablet, this, tablet_id, std::ref(handle));
ASSERT_TRUE(!handle.is_valid());
tx_data.reset();
ret = tablet->get_tx_data(tx_data);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(ObTabletStatus::CREATING, tx_data.tablet_status_);
// tablet status NORMAL
ObMulSourceDataNotifyArg trans_flags;
trans_flags.tx_id_ = 1;
trans_flags.scn_ = share::SCN::minus(share::SCN::max_scn(), 1);
trans_flags.for_replay_ = for_replay;
ret = ObTabletCreateDeleteHelper::set_tablet_final_status(tablet_handle, ObTabletStatus::NORMAL,
trans_flags.scn_, share::SCN::max_scn(), trans_flags.for_replay_);
ASSERT_EQ(OB_SUCCESS, ret);
t1.join();
ASSERT_TRUE(handle.is_valid());
ASSERT_EQ(tablet_id, handle.get_obj()->tablet_meta_.tablet_id_);
tx_data.reset();
ret = tablet->get_tx_data(tx_data);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(ObTabletStatus::NORMAL, tx_data.tablet_status_);
ASSERT_EQ(ObTabletCommon::FINAL_TX_ID, tx_data.tx_id_);
// tablet status DELETING
tx_data.tx_id_ = 2;
tx_data.tx_scn_ = share::SCN::max_scn();
tx_data.tablet_status_ = ObTabletStatus::DELETING;
ret = tablet_handle.get_obj()->set_tx_data(tx_data, for_replay);
ASSERT_EQ(OB_SUCCESS, ret);
// get tablet
ret = ls_tablet_service.get_tablet(tablet_id, fail_tablet_handle, timeout_us);
ASSERT_EQ(OB_TIMEOUT, ret);
handle.reset();
std::thread t2(&TestTabletStatus::wait_for_tablet, this, tablet_id, std::ref(handle));
ASSERT_TRUE(!handle.is_valid());
tx_data.reset();
ret = tablet->get_tx_data(tx_data);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(ObTabletStatus::DELETING, tx_data.tablet_status_);
// tablet status DELETED
trans_flags.tx_id_ = 2;
ret = ObTabletCreateDeleteHelper::set_tablet_final_status(tablet_handle, ObTabletStatus::DELETED,
trans_flags.scn_, share::SCN::max_scn(), trans_flags.for_replay_);
ASSERT_EQ(OB_SUCCESS, ret);
t2.join();
ASSERT_TRUE(!handle.is_valid()); // tablet is deleted, so handle is invalid
tx_data.reset();
ret = tablet->get_tx_data(tx_data);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(ObTabletStatus::DELETED, tx_data.tablet_status_);
// get tablet
ret = ls_tablet_service.get_tablet(tablet_id, tablet_handle);
ASSERT_EQ(OB_TABLET_NOT_EXIST, ret);
}
} // namespace storage
} // namespace oceanbase
int main(int argc, char **argv)
{
system("rm -f test_tablet_status.log*");
OB_LOGGER.set_file_name("test_tablet_status.log", true);
OB_LOGGER.set_log_level("INFO");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

2
mittest/mtlenv/storage/test_tenant_meta_mem_mgr.cpp
View File

@ -354,7 +354,6 @@ void TestConcurrentT3M::run1()
if (0 == count % N) { /* mock empty tablet to del successfully */
tablet->table_store_addr_.addr_.set_none_addr();
tablet->storage_schema_addr_.addr_.set_none_addr();
tablet->mds_data_.auto_inc_seq_.addr_.set_none_addr();
tablet->rowkey_read_info_ = nullptr;
}
@ -647,7 +646,6 @@ TEST_F(TestTenantMetaMemMgr, test_tablet)
/* mock empty tablet to del successfully */
tablet->table_store_addr_.addr_.set_none_addr();
tablet->storage_schema_addr_.addr_.set_none_addr();
tablet->mds_data_.auto_inc_seq_.addr_.set_none_addr();
tablet->rowkey_read_info_ = nullptr;
tablet->tablet_meta_.ls_id_ = key.ls_id_;

3
mittest/simple_server/errsim/storage_ha/errsim_test_transfer_handler.cpp
View File

@ -113,9 +113,10 @@ int TestTransferHandler::gen_mock_data(const ObTransferTaskID task_id, const ObT
finish_scn.convert_for_inner_table_field(1666844202208490);
ObCurTraceId::TraceId trace_id;
trace_id.init(GCONF.self_addr_);
uint64_t data_version = 0;
ret = task.init(task_id, src_ls, dest_ls, ObString::make_string("500016:500014"), ObString("500030:500031"), ObString("500016:500015"),
ObString::make_string("1152921504606846983"), ObString::make_string("1152921504606846983:0"), start_scn, finish_scn, status, trace_id, OB_SUCCESS,
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id);
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id, data_version);
return ret;
}

3
mittest/simple_server/storage_ha/test_transfer_and_restart_basic.cpp
View File

@ -109,9 +109,10 @@ int TestTransferHandler::gen_mock_data(const ObTransferTaskID task_id, const ObT
owner_id.convert_from_value(999);
ObCurTraceId::TraceId trace_id;
trace_id.init(GCONF.self_addr_);
uint64_t data_version = 0;
ret = task.init(task_id, src_ls, dest_ls, ObString::make_string("500016:500014"), ObString("500030:500031"), ObString("500016:500015"),
ObString::make_string("1152921504606846983"), ObString::make_string("1152921504606846983:0"), start_scn, finish_scn, status, trace_id, OB_SUCCESS,
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id);
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id, data_version);
return ret;
}

3
mittest/simple_server/storage_ha/test_transfer_complete_restart_with_mds_flush.cpp
View File

@ -109,9 +109,10 @@ int TestTransferHandler::gen_mock_data(const ObTransferTaskID task_id, const ObT
owner_id.convert_from_value(999);
ObCurTraceId::TraceId trace_id;
trace_id.init(GCONF.self_addr_);
uint64_t data_version = 0;
ret = task.init(task_id, src_ls, dest_ls, ObString::make_string("500016:500014"), ObString("500030:500031"), ObString("500016:500015"),
ObString::make_string("1152921504606846983"), ObString::make_string("1152921504606846983:0"), start_scn, finish_scn, status, trace_id, OB_SUCCESS,
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id);
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id, data_version);
return ret;
}

3
mittest/simple_server/storage_ha/test_transfer_complete_restart_without_mds_flush.cpp
View File

@ -109,9 +109,10 @@ int TestTransferHandler::gen_mock_data(const ObTransferTaskID task_id, const ObT
owner_id.convert_from_value(999);
ObCurTraceId::TraceId trace_id;
trace_id.init(GCONF.self_addr_);
uint64_t data_version = 0;
ret = task.init(task_id, src_ls, dest_ls, ObString::make_string("500016:500014"), ObString("500030:500031"), ObString("500016:500015"),
ObString::make_string("1152921504606846983"), ObString::make_string("1152921504606846983:0"), start_scn, finish_scn, status, trace_id, OB_SUCCESS,
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id);
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id, data_version);
return ret;
}

3
mittest/simple_server/storage_ha/test_transfer_doing_stage_restart_with_mds_flush.cpp
View File

@ -108,9 +108,10 @@ int TestTransferHandler::gen_mock_data(const ObTransferTaskID task_id, const ObT
owner_id.convert_from_value(999);
ObCurTraceId::TraceId trace_id;
trace_id.init(GCONF.self_addr_);
uint64_t data_version = 0;
ret = task.init(task_id, src_ls, dest_ls, ObString::make_string("500016:500014"), ObString("500030:500031"), ObString("500016:500015"),
ObString::make_string("1152921504606846983"), ObString::make_string("1152921504606846983:0"), start_scn, finish_scn, status, trace_id, OB_SUCCESS,
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id);
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id, data_version);
return ret;
}

3
mittest/simple_server/storage_ha/test_transfer_doing_stage_restart_without_mds_flush.cpp
View File

@ -108,9 +108,10 @@ int TestTransferHandler::gen_mock_data(const ObTransferTaskID task_id, const ObT
owner_id.convert_from_value(999);
ObCurTraceId::TraceId trace_id;
trace_id.init(GCONF.self_addr_);
uint64_t data_version = 0;
ret = task.init(task_id, src_ls, dest_ls, ObString::make_string("500016:500014"), ObString("500030:500031"), ObString("500016:500015"),
ObString::make_string("1152921504606846983"), ObString::make_string("1152921504606846983:0"), start_scn, finish_scn, status, trace_id, OB_SUCCESS,
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id);
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id, data_version);
return ret;
}

3
mittest/simple_server/storage_ha/test_transfer_handler.cpp
View File

@ -109,9 +109,10 @@ int TestTransferHandler::gen_mock_data(const ObTransferTaskID task_id, const ObT
owner_id.convert_from_value(999);
ObCurTraceId::TraceId trace_id;
trace_id.init(GCONF.self_addr_);
uint64_t data_version = 0;
ret = task.init(task_id, src_ls, dest_ls, ObString::make_string("500016:500014"), ObString("500030:500031"), ObString("500016:500015"),
ObString::make_string("1152921504606846983"), ObString::make_string("1152921504606846983:0"), start_scn, finish_scn, status, trace_id, OB_SUCCESS,
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id);
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id, data_version);
return ret;
}

3
mittest/simple_server/storage_ha/test_transfer_start_stage_restart_without_mds_flush.cpp
View File

@ -108,9 +108,10 @@ int TestTransferHandler::gen_mock_data(const ObTransferTaskID task_id, const ObT
owner_id.convert_from_value(999);
ObCurTraceId::TraceId trace_id;
trace_id.init(GCONF.self_addr_);
uint64_t data_version = 0;
ret = task.init(task_id, src_ls, dest_ls, ObString::make_string("500016:500014"), ObString("500030:500031"), ObString("500016:500015"),
ObString::make_string("1152921504606846983"), ObString::make_string("1152921504606846983:0"), start_scn, finish_scn, status, trace_id, OB_SUCCESS,
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id);
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id, data_version);
return ret;
}

3
mittest/simple_server/storage_ha/test_transfer_with_empty_shell.cpp
View File

@ -109,9 +109,10 @@ int TestTransferHandler::gen_mock_data(const ObTransferTaskID task_id, const ObT
owner_id.convert_from_value(999);
ObCurTraceId::TraceId trace_id;
trace_id.init(GCONF.self_addr_);
uint64_t data_version = 0;
ret = task.init(task_id, src_ls, dest_ls, ObString::make_string("500016:500014"), ObString("500030:500031"), ObString("500016:500015"),
ObString::make_string("1152921504606846983"), ObString::make_string("1152921504606846983:0"), start_scn, finish_scn, status, trace_id, OB_SUCCESS,
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id);
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id, data_version);
return ret;
}

17
mittest/simple_server/test_mds_transaction.cpp
View File

@ -247,10 +247,11 @@ TEST_F(TestMdsTransactionTest, test_mds_table_gc_and_recycle)
std::this_thread::sleep_for(std::chrono::seconds(15));
ASSERT_EQ(false, static_cast<ObTabletPointer*>(tablet_handle.get_obj()->pointer_hdl_.get_resource_ptr())->mds_table_handler_.mds_table_handle_.is_valid());
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle));// 重新获取一下tablet handle
ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->get_mds_data_from_tablet<ObTabletBindingMdsUserData>([&data_to_write](const ObTabletBindingMdsUserData &data_to_read) -> int {
OB_ASSERT(data_to_write.schema_version_ == data_to_read.schema_version_);
return OB_SUCCESS;
}));
ASSERT_EQ(OB_SUCCESS, (tablet_handle.get_obj()->get_mds_data_from_tablet<mds::DummyKey, ObTabletBindingMdsUserData>(mds::DummyKey(), share::SCN::max_scn(), 1_s,
[&data_to_write](const ObTabletBindingMdsUserData &data_to_read) -> int {
OB_ASSERT(data_to_write.schema_version_ == data_to_read.schema_version_);
return OB_SUCCESS;
})));
mock_tablet_oldest_scn = unittest::mock_scn(1000);
}
}
@ -293,19 +294,19 @@ TEST_F(TestMdsTransactionTest, test_mds_table_get_tablet_status_transfer_in_writ
storage::ObTabletHandle tablet_handle;
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle));
// 6. 调用tablet接口写入多源数据，提交
share::SCN max_decided_scn;
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_max_decided_scn(max_decided_scn));
MdsCtx ctx1(mds::MdsWriter(ObTransID(1)));
share::SCN rec_scn;
ASSERT_EQ(OB_STATE_NOT_MATCH, tablet_handle.get_obj()->check_transfer_in_redo_written(written));// 这个时候因为tablet status不是TRANSFER IN所以查不出来
ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->set(data_to_write, ctx1));
ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->check_transfer_in_redo_written(written));// 这个时候tablet status是TRANSFER IN, 但事务还没写日志，所以可以查出结果，但结果是false
ASSERT_EQ(false, written);
ctx1.single_log_commit(mock_scn(10), mock_scn(10000000));
ctx1.single_log_commit(max_decided_scn, max_decided_scn);
ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->check_transfer_in_redo_written(written));// 这个时候tablet status是TRANSFER IN, 并且事务已经提交，所以可以查出结果，并且结果是true
ASSERT_EQ(true, written);
ASSERT_EQ(true, static_cast<ObTabletPointer*>(tablet_handle.get_obj()->pointer_hdl_.get_resource_ptr())->mds_table_handler_.mds_table_handle_.is_valid());
std::this_thread::sleep_for(std::chrono::seconds(5));
share::SCN max_decided_scn;
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_max_decided_scn(max_decided_scn));
ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->mds_table_flush(max_decided_scn));
// 7. 检查mds table的存在情况
std::this_thread::sleep_for(std::chrono::seconds(5));
@ -361,4 +362,4 @@ int main(int argc, char **argv)
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
#endif
#endif

8
mittest/simple_server/test_tenant_transfer_service.cpp
View File

@ -116,9 +116,10 @@ int TestTenantTransferService::gen_mock_data(const ObTransferTaskID task_id, con
finish_scn.convert_for_inner_table_field(1666844202208490);
ObCurTraceId::TraceId trace_id;
trace_id.init(GCONF.self_addr_);
uint64_t data_version = 0;
ret = task.init(task_id, src_ls, dest_ls, ObString::make_string("500016:500014"), ObString("500030:500031"), ObString("500016:500015"),
ObString::make_string("1152921504606846983"), ObString::make_string("1152921504606846983:0"), start_scn, finish_scn, status, trace_id, OB_SUCCESS,
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id);
ObTransferTaskComment::EMPTY_COMMENT, ObBalanceTaskID(123), owner_id, data_version);
return ret;
}
@ -186,6 +187,7 @@ TEST_F(TestTenantTransferService, test_service)
ARRAY_FOREACH(g_part_list, idx) {
ASSERT_TRUE(is_contain(task.get_part_list(), g_part_list.at(idx)));
}
ASSERT_TRUE(task.get_data_version() > 0);
LOG_INFO("generate transfer task", K(task));
// generate tablet_list
@ -252,7 +254,6 @@ TEST_F(TestTenantTransferService, test_service)
ObTransferTask history_task;
ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get_history_task(inner_sql_proxy, g_tenant_id, task_id, history_task, create_time, finish_time));
ASSERT_TRUE(history_task.get_status().is_completed_status());
// test retry task with interval
sql.reset();
ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("alter system set _transfer_task_retry_interval = '1h'"));
@ -277,6 +278,9 @@ TEST_F(TestTenantTransferService, test_service)
ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("alter system set _transfer_task_retry_interval = '0'"));
ASSERT_EQ(OB_SUCCESS, inner_sql_proxy.write(g_tenant_id, sql.ptr(), affected_rows));
ASSERT_TRUE(OB_NEED_RETRY != tenant_transfer->process_init_task_(retry_task_id));
uint64_t data_version = 0;
ASSERT_EQ(OB_SUCCESS, ObShareUtil::fetch_current_data_version(inner_sql_proxy, g_tenant_id, data_version));
ASSERT_TRUE(data_version == history_task.get_data_version());
}
TEST_F(TestTenantTransferService, test_batch_part_list)

55
mittest/simple_server/test_transfer_task_operator.cpp
View File

@ -16,6 +16,8 @@
#include <gmock/gmock.h>
#include "env/ob_simple_cluster_test_base.h"
#include "lib/ob_errno.h"
#define private public
#include "share/transfer/ob_transfer_task_operator.h"
namespace oceanbase
@ -47,6 +49,7 @@ public:
ObCurTraceId::TraceId trace_id_;
ObTransferTask task_;
transaction::tablelock::ObTableLockOwnerID lock_owner_id_;
uint64_t data_version_;
};
void TestTransferTaskOperator::SetUp()
@ -61,6 +64,7 @@ void TestTransferTaskOperator::SetUp()
ObString trace_id_str = "YCDC56458724D-0005EBECD3F9DB9D-0-0";
trace_id_.parse_from_buf(trace_id_str.ptr());
lock_owner_id_.convert_from_value(999);
data_version_ = 0;
for(int64_t i = 0; i < 100; ++i) {
ObTransferPartInfo part;
ObTransferTabletInfo tablet;
@ -76,7 +80,7 @@ void TestTransferTaskOperator::SetUp()
}
}
ASSERT_EQ(OB_SUCCESS, task_.init(task_id_, src_ls_, dest_ls_, part_list_, status_, trace_id_,
ObBalanceTaskID(12)));
ObBalanceTaskID(12), data_version_));
LOG_INFO("tranfer task init", K(task_));
}
@ -182,7 +186,7 @@ TEST_F(TestTransferTaskOperator, test_basic_func)
ASSERT_TRUE(!task.is_valid());
ASSERT_EQ(OB_SUCCESS, task.init(task_id_, src_ls_, dest_ls_, part_list_str, empty_str, empty_str, table_lock_tablet_list_str, tablet_list_str,
start_scn_, finish_scn_, status_, trace_id_, OB_SUCCESS, ObTransferTaskComment::EMPTY_COMMENT,
ObBalanceTaskID(2), lock_owner_id_));
ObBalanceTaskID(2), lock_owner_id_, data_version_));
LOG_INFO("tranfer task other init", K(task));
ASSERT_TRUE(task.is_valid());
task.reset();
@ -232,7 +236,7 @@ TEST_F(TestTransferTaskOperator, test_operator)
ObTransferTask other_task;
ObTransferTaskID other_task_id(222);
ASSERT_EQ(OB_SUCCESS, other_task.init(other_task_id, ObLSID(1003), ObLSID(1004), part_list_,
ObTransferStatus(ObTransferStatus::INIT), trace_id_, ObBalanceTaskID(2)));
ObTransferStatus(ObTransferStatus::INIT), trace_id_, ObBalanceTaskID(2), data_version_));
ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::insert(sql_proxy, tenant_id_, other_task));
ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::insert(sql_proxy, tenant_id_, task_));
ASSERT_EQ(OB_ENTRY_EXIST, ObTransferTaskOperator::insert(sql_proxy, tenant_id_, task_));
@ -276,7 +280,7 @@ TEST_F(TestTransferTaskOperator, test_operator)
ASSERT_EQ(OB_ENTRY_NOT_EXIST, ObTransferTaskOperator::get_by_dest_ls(sql_proxy, tenant_id_, ObLSID(555), task, 0/*group_id*/));
ObTransferTask dup_dest_ls_task;
ASSERT_EQ(OB_SUCCESS, dup_dest_ls_task.init(ObTransferTaskID(2223), ObLSID(1003), ObLSID(1004),
part_list_, ObTransferStatus(ObTransferStatus::INIT), trace_id_, ObBalanceTaskID(2)));
part_list_, ObTransferStatus(ObTransferStatus::INIT), trace_id_, ObBalanceTaskID(2), data_version_));
ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::insert(sql_proxy, tenant_id_, dup_dest_ls_task));
task.reset();
ASSERT_EQ(OB_ERR_UNEXPECTED, ObTransferTaskOperator::get_by_dest_ls(sql_proxy, tenant_id_, ObLSID(1004), task, 0/*group_id*/));
@ -395,6 +399,49 @@ TEST_F(TestTransferTaskOperator, test_operator)
ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get_last_task_by_balance_task_id(sql_proxy, tenant_id_, task_.get_balance_task_id(), last_task, finish_time));
ASSERT_TRUE(last_task.is_valid() && last_task.get_task_id() == task_id_);
ASSERT_TRUE(finish_time > 0);
// test convert data version
ObString empty_string("");
ASSERT_TRUE(empty_string.empty() && empty_string == ObString());
uint64_t data_version = 0;
bool data_version_not_exist = false;
bool is_history = true;
bool not_history = false;
// a. empty str
ObString data_version_str;
ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::convert_data_version_(tenant_id_, is_history, data_version_not_exist, data_version_str, data_version));
ASSERT_TRUE(DEFAULT_MIN_DATA_VERSION == data_version);
data_version = 0;
ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::convert_data_version_(tenant_id_, not_history, data_version_not_exist, data_version_str, data_version));
ASSERT_TRUE(DEFAULT_MIN_DATA_VERSION == data_version);
// b. valid str
char version[common::OB_CLUSTER_VERSION_LENGTH] = {'\0'};
int64_t len = ObClusterVersion::print_version_str(
version, common::OB_CLUSTER_VERSION_LENGTH, DATA_VERSION_4_3_2_0);
data_version_str = ObString::make_string(version);
ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::convert_data_version_(tenant_id_, is_history, data_version_not_exist, data_version_str, data_version));
ASSERT_TRUE(DATA_VERSION_4_3_2_0 == data_version);
data_version = 0;
ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::convert_data_version_(tenant_id_, not_history, data_version_not_exist, data_version_str, data_version));
ASSERT_TRUE(DATA_VERSION_4_3_2_0 == data_version);
// c. default str (with and without column)
data_version_str = ObString::make_string("");
data_version_not_exist = true;
ASSERT_EQ(OB_NEED_RETRY, ObTransferTaskOperator::convert_data_version_(tenant_id_, is_history, data_version_not_exist, data_version_str, data_version));
ASSERT_EQ(OB_NEED_RETRY, ObTransferTaskOperator::convert_data_version_(tenant_id_, not_history, data_version_not_exist, data_version_str, data_version));
ObMySQLProxy &inner_sql_proxy = get_curr_observer().get_mysql_proxy();
ObSqlString sql;
int64_t affected_rows = 0;
ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("delete from oceanbase.__all_column where tenant_id=0 and table_id=%lu and column_name='data_version'", OB_ALL_TRANSFER_TASK_HISTORY_TID));
ASSERT_EQ(OB_SUCCESS, inner_sql_proxy.write(tenant_id_, sql.ptr(), affected_rows));
data_version = 0;
ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::convert_data_version_(tenant_id_, is_history, data_version_not_exist, data_version_str, data_version));
ASSERT_TRUE(DEFAULT_MIN_DATA_VERSION == data_version);
ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("delete from oceanbase.__all_column where tenant_id=0 and table_id=%lu and column_name='data_version'", OB_ALL_TRANSFER_TASK_TID));
ASSERT_EQ(OB_SUCCESS, inner_sql_proxy.write(tenant_id_, sql.ptr(), affected_rows));
data_version = 0;
ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::convert_data_version_(tenant_id_, not_history, data_version_not_exist, data_version_str, data_version));
ASSERT_TRUE(DEFAULT_MIN_DATA_VERSION == data_version);
}
} // namespace share

2
src/logservice/libobcdc/src/ob_cdc_multi_data_source_info.cpp
View File

@ -52,7 +52,7 @@ int MultiDataSourceNode::init(
common::ObString data;
data.assign_ptr(reinterpret_cast<const char *>(buf), buf_size);
if (OB_FAIL(tx_buf_node_.init(type, data, share::SCN(), nullptr))) {
if (OB_FAIL(tx_buf_node_.init(type, data, share::SCN(), transaction::ObTxSEQ::mk_v0(1), nullptr))) {
LOG_ERROR("init tx_buf_node failed", KR(ret), K(lsn), K(type), K(data), K(buf_size));
}
}

11
src/logservice/libobcdc/src/ob_cdc_multi_data_source_info.h
View File

@ -15,12 +15,11 @@
#ifndef OCEANBASE_LIBOBCDC_MULTI_DATA_SOURCE_INFO_H_
#define OCEANBASE_LIBOBCDC_MULTI_DATA_SOURCE_INFO_H_
#include "logservice/palf/lsn.h" // palf::LSN
#include "share/ls/ob_ls_operator.h" // share::ObLSAttr
#include "storage/tx/ob_multi_data_source.h" // transaction::ObTxBufferNode
#include "ob_cdc_tablet_to_table_info.h" // CDCTabletChangeInfo
#include "logservice/data_dictionary/ob_data_dict_struct.h" // ObDictTenantMeta
#include "logservice/palf/lsn.h" // palf::LSN
#include "share/ls/ob_ls_operator.h" // share::ObLSAttr
#include "storage/tx/ob_multi_data_source_tx_buffer_node.h" // transaction::ObTxBufferNode
#include "ob_cdc_tablet_to_table_info.h" // CDCTabletChangeInfo
#include "logservice/data_dictionary/ob_data_dict_struct.h" // ObDictTenantMeta
namespace oceanbase
{

12
src/logservice/libobcdc/src/ob_cdc_tablet_to_table_info.h
View File

@ -15,12 +15,12 @@
#ifndef OCEANBASE_LIBOBCDC_TABLET_TO_TABLE_INFO_H__
#define OCEANBASE_LIBOBCDC_TABLET_TO_TABLE_INFO_H__
#include "common/ob_tablet_id.h" // common::ObTabletID
#include "lib/hash/ob_linear_hash_map.h" // ObLinkHashMap
#include "share/schema/ob_schema_struct.h" // ObTableType
#include "storage/tx/ob_multi_data_source.h" // ObTxBufferNode
#include "rootserver/ob_tablet_creator.h" // ObBatchCreateTabletArg
#include "rootserver/ob_tablet_drop.h" // ObBatchRemoveTabletArg
#include "common/ob_tablet_id.h" // common::ObTabletID
#include "lib/hash/ob_linear_hash_map.h" // ObLinkHashMap
#include "share/schema/ob_schema_struct.h" // ObTableType
#include "storage/tx/ob_multi_data_source_tx_buffer_node.h" // ObTxBufferNode
#include "rootserver/ob_tablet_creator.h" // ObBatchCreateTabletArg
#include "rootserver/ob_tablet_drop.h" // ObBatchRemoveTabletArg
#include "logservice/common_util/ob_log_ls_define.h"
namespace oceanbase

17
src/logservice/ob_log_handler.cpp
View File

@ -1776,5 +1776,22 @@ bool ObLogHandler::is_offline() const
return true == ATOMIC_LOAD(&is_offline_);
}
int ObLogHandler::is_replay_fatal_error(bool &has_fatal_error)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
} else if (true == is_in_stop_state_) {
ret = OB_NOT_RUNNING;
} else {
RLockGuard guard(lock_);
ObLSID ls_id(id_);
if (OB_FAIL(replay_service_->has_fatal_error(ls_id, has_fatal_error))) {
CLOG_LOG(WARN, "has_fatal_error failed", KR(ret), K(ls_id));
}
}
return ret;
}
} // end namespace logservice
} // end napespace oceanbase

8
src/logservice/ob_log_handler.h
View File

@ -198,6 +198,7 @@ public:
virtual int offline() = 0;
virtual int online(const palf::LSN &lsn, const share::SCN &scn) = 0;
virtual bool is_offline() const = 0;
virtual int is_replay_fatal_error(bool &has_fatal_error) = 0;
};
class ObLogHandler : public ObILogHandler, public ObLogHandlerBase
@ -752,6 +753,13 @@ public:
int offline() override final;
int online(const palf::LSN &lsn, const share::SCN &scn) override final;
bool is_offline() const override final;
// @brief: check there's a fatal error in replay service.
// @param[out] has_fatal_error.
// @return:
// OB_NOT_INIT: not inited
// OB_NOT_RUNNING: in stop state
// OB_EAGAIN: try lock failed, need retry.
int is_replay_fatal_error(bool &has_fatal_error);
private:
static constexpr int64_t MIN_CONN_TIMEOUT_US = 5 * 1000 * 1000; // 5s
const int64_t MAX_APPEND_RETRY_INTERNAL = 500 * 1000L;

24
src/logservice/replayservice/ob_log_replay_service.cpp
View File

@ -702,6 +702,30 @@ void ObLogReplayService::free_replay_task_log_buf(ObLogReplayTask *task)
}
}
int ObLogReplayService::has_fatal_error(const ObLSID &ls_id,
bool &bool_ret)
{
int ret = OB_SUCCESS;
ObReplayStatus *replay_status = NULL;
ObReplayStatusGuard guard;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
CLOG_LOG(WARN, "replay service not init", K(ret));
} else if (OB_FAIL(get_replay_status_(ls_id, guard))) {
CLOG_LOG(WARN, "guard get replay status failed", K(ret), K(ls_id));
} else if (NULL == (replay_status = guard.get_replay_status())) {
ret = OB_ERR_UNEXPECTED;
CLOG_LOG(WARN, "replay status is not exist", K(ret), K(ls_id));
} else if (replay_status->try_rdlock()){
bool_ret = replay_status->has_fatal_error();
replay_status->unlock();
} else {
ret = OB_EAGAIN;
CLOG_LOG(WARN, "try_rdlock failed", K(ret), K(ls_id));
}
return ret;
}
void ObLogReplayService::free_replay_log_buf_(ObLogReplayBuffer *&replay_log_buf)
{
allocator_->free_replay_log_buf(replay_log_buf);

2
src/logservice/replayservice/ob_log_replay_service.h
View File

@ -166,6 +166,8 @@ public:
void *alloc_replay_task(const int64_t size);
void free_replay_task(ObLogReplayTask *task);
void free_replay_task_log_buf(ObLogReplayTask *task);
int has_fatal_error(const share::ObLSID &ls_id,
bool &bool_ret);
private:
int get_replay_status_(const share::ObLSID &id,
ObReplayStatusGuard &guard);

24
src/logservice/replayservice/ob_tablet_replay_executor.cpp
View File

@ -20,6 +20,8 @@ namespace oceanbase
namespace logservice
{
ERRSIM_POINT_DEF(EN_REPLAY_FATAL_ERROR);
#ifdef CLOG_LOG_LIMIT
#undef CLOG_LOG_LIMIT
#endif
@ -111,6 +113,28 @@ int ObTabletReplayExecutor::execute(const share::SCN &scn, const share::ObLSID &
}
}
#ifdef ERRSIM
if (OB_SUCC(ret)) {
const int64_t errsim_migration_ls_id = GCONF.errsim_migration_ls_id;
const ObLSID errsim_ls_id(errsim_migration_ls_id);
const ObString &errsim_migration_dest_server_addr = GCONF.errsim_migration_dest_server_addr.str();
common::ObAddr addr;
const ObAddr &my_addr = GCONF.self_addr_;
if (!errsim_migration_dest_server_addr.empty() && OB_FAIL(addr.parse_from_string(errsim_migration_dest_server_addr))) {
CLOG_LOG(WARN, "failed to parse from string to addr", K(ret), K(errsim_migration_dest_server_addr));
} else {
if (ls_id == errsim_ls_id && my_addr == addr) {
ret = EN_REPLAY_FATAL_ERROR ? : OB_SUCCESS;
if (OB_FAIL(ret)) {
STORAGE_LOG(ERROR, "fake EN_REPLAY_FATAL_ERROR", K(ret));
}
}
}
}
#endif
return ret;
}

9
src/observer/ob_inner_sql_connection.cpp
View File

@ -1126,8 +1126,13 @@ int ObInnerSQLConnection::register_multi_data_source(const uint64_t &tenant_id,
} else {
MTL_SWITCH(tenant_id)
{
if (OB_FAIL(MTL(transaction::ObTransService *)
->register_mds_into_tx(*tx_desc, ls_id, type, buf, buf_len, 0, register_flag))) {
if (OB_FAIL(MTL(transaction::ObTransService *)->register_mds_into_tx(*tx_desc,
ls_id,
type,
buf,
buf_len,
0,
register_flag))) {
LOG_WARN("regiser multi data source failed", K(ret), K(tenant_id), K(type));
} else if (OB_FAIL(res.close())) {
LOG_WARN("close result set failed", K(ret), K(tenant_id));

19
src/observer/ob_inner_sql_connection.h
View File

@ -28,6 +28,10 @@
namespace oceanbase
{
namespace obrpc
{
class ObInnerSqlRpcP;
}
namespace common
{
class ObString;
@ -92,6 +96,7 @@ class ObInnerSQLConnection
: public common::sqlclient::ObISQLConnection,
public common::ObDLinkBase<ObInnerSQLConnection>
{
friend class obrpc::ObInnerSqlRpcP;
public:
static constexpr const char LABEL[] = "RPInnerSqlConn";
class SavedValue
@ -166,12 +171,6 @@ public:
const ObTimeZoneInfo *tz_info,
ObObj *result) override;
virtual int start_transaction(const uint64_t &tenant_id, bool with_snap_shot = false) override;
virtual int register_multi_data_source(const uint64_t &tenant_id,
const share::ObLSID ls_id,
const transaction::ObTxDataSourceType type,
const char *buf,
const int64_t buf_len,
const transaction::ObRegisterMdsFlag &register_flag = transaction::ObRegisterMdsFlag());
virtual sqlclient::ObCommonServerConnectionPool *get_common_server_pool() override;
virtual int rollback() override;
virtual int commit() override;
@ -270,6 +269,14 @@ public:
// set timeout to session variable
int set_session_timeout(int64_t query_timeout, int64_t trx_timeout);
public:// for mds
int register_multi_data_source(const uint64_t &tenant_id,
const share::ObLSID ls_id,
const transaction::ObTxDataSourceType type,
const char *buf,
const int64_t buf_len,
const transaction::ObRegisterMdsFlag &register_flag = transaction::ObRegisterMdsFlag());
public:
static int process_record(sql::ObResultSet &result_set,
sql::ObSqlCtx &sql_ctx,

9
src/observer/ob_inner_sql_rpc_processor.cpp
View File

@ -64,9 +64,12 @@ int ObInnerSqlRpcP::process_register_mds(sqlclient::ObISQLConnection *conn,
int64_t pos = 0;
if (OB_FAIL(mds_str.deserialize(arg.get_inner_sql().ptr(), arg.get_inner_sql().length(), pos))) {
LOG_WARN("deserialize multi data source str failed", K(ret), K(arg), K(pos));
} else if (OB_FAIL(inner_conn->register_multi_data_source(
arg.get_tenant_id(), mds_str.get_ls_id(), mds_str.get_msd_type(),
mds_str.get_msd_buf(), mds_str.get_msd_buf_len(), mds_str.get_register_flag()))) {
} else if (OB_FAIL(inner_conn->register_multi_data_source(arg.get_tenant_id(),
mds_str.get_ls_id(),
mds_str.get_msd_type(),
mds_str.get_msd_buf(),
mds_str.get_msd_buf_len(),
mds_str.get_register_flag()))) {
LOG_WARN("register multi data source failed", K(ret), K(arg.get_tenant_id()), K(mds_str));
}

11
src/observer/ob_rpc_processor_simple.cpp
View File

@ -2465,9 +2465,14 @@ int ObRegisterTxDataP::process()
if (OB_ISNULL(tx_svc)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null tx service ptr", KR(ret), K(arg_));
} else if (OB_FAIL(tx_svc->register_mds_into_tx(*(arg_.tx_desc_), arg_.ls_id_, arg_.type_,
arg_.buf_.ptr(), arg_.buf_.length(),
arg_.request_id_, arg_.register_flag_))) {
} else if (OB_FAIL(tx_svc->register_mds_into_tx(*(arg_.tx_desc_),
arg_.ls_id_,
arg_.type_,
arg_.buf_.ptr(),
arg_.buf_.length(),
arg_.request_id_,
arg_.register_flag_,
arg_.seq_no_))) {
LOG_WARN("register into tx failed", KR(ret), K(arg_));
} else if (OB_FAIL(tx_svc->collect_tx_exec_result(*(arg_.tx_desc_), result_.tx_result_))) {
LOG_WARN("collect tx result failed", KR(ret), K(result_));

7
src/observer/ob_server.cpp
View File

@ -88,6 +88,7 @@
#include "share/ob_tablet_autoincrement_service.h"
#include "share/ob_tenant_mem_limit_getter.h"
#include "storage/slog_ckpt/ob_server_checkpoint_slog_handler.h"
#include "storage/tablet/ob_mds_schema_helper.h"
#include "storage/tx_storage/ob_tenant_freezer.h"
#include "storage/tx_storage/ob_tenant_memory_printer.h"
#include "storage/compaction/ob_compaction_diagnose.h"
@ -916,6 +917,12 @@ int ObServer::start()
FLOG_INFO("success to start block manager");
}
if (FAILEDx(ObMdsSchemaHelper::get_instance().init())) {
LOG_ERROR("fail to init mds schema helper", K(ret));
} else {
FLOG_INFO("success to init mds schema helper");
}
if (FAILEDx(ObIOManager::get_instance().start())) {
LOG_ERROR("fail to start io manager", KR(ret));
} else {

2
src/observer/virtual_table/ob_all_virtual_mds_event_history.cpp
View File

@ -301,7 +301,7 @@ int ObAllVirtualMdsEventHistory::convert_event_info_to_row_(const MdsEventKey &k
break;
}
case OB_APP_MIN_COLUMN_ID + 15: {// seq_no
cur_row_.cells_[i].set_int(event.seq_no_);
cur_row_.cells_[i].set_int(event.seq_no_.get_seq());
break;
}
case OB_APP_MIN_COLUMN_ID + 16: {// redo_scn

2
src/observer/virtual_table/ob_all_virtual_mds_node_stat.cpp
View File

@ -174,7 +174,7 @@ int ObAllVirtualMdsNodeStat::convert_node_info_to_row_(const storage::mds::MdsNo
break;
}
case OB_APP_MIN_COLUMN_ID + 10: {// seq_no
cur_row_.cells_[i].set_int(node_info.seq_no_);
cur_row_.cells_[i].set_int(node_info.seq_no_.cast_to_int());
break;
}
case OB_APP_MIN_COLUMN_ID + 11: {// redo_scn

1
src/observer/virtual_table/ob_all_virtual_tablet_info.cpp
View File

@ -154,7 +154,6 @@ int ObAllVirtualTabletInfo::process_curr_tenant(ObNewRow *&row)
SERVER_LOG(WARN, "tablet should not null", K(ret), K(tablet_handle));
} else if (OB_FAIL(tablet->ObITabletMdsInterface::get_latest_tablet_status(latest_user_data, is_committed))) {
if (OB_EMPTY_RESULT == ret || OB_ERR_SHARED_LOCK_CONFLICT == ret) {
is_committed = false;
is_empty_result = true;
ret = OB_SUCCESS;
} else {

19
src/observer/virtual_table/ob_all_virtual_tablet_sstable_macro_info.cpp
View File

@ -14,6 +14,7 @@
#include "sql/session/ob_sql_session_info.h"
#include "storage/blocksstable/ob_logic_macro_id.h"
#include "storage/meta_mem/ob_tenant_meta_mem_mgr.h"
#include "storage/tablet/ob_mds_schema_helper.h"
#include "sql/session/ob_sql_session_info.h"
namespace oceanbase
@ -257,6 +258,13 @@ int ObAllVirtualTabletSSTableMacroInfo::get_macro_info(
info.store_range_))) {
SERVER_LOG(WARN, "fail to get store range", K(ret), K(macro_desc.range_));
}
} else if (curr_sstable_->is_mds_sstable()) {
const storage::ObITableReadInfo *index_read_info = storage::ObMdsSchemaHelper::get_instance().get_rowkey_read_info();
if (OB_FAIL(macro_desc.range_.to_store_range(index_read_info->get_columns_desc(),
rowkey_allocator_,
info.store_range_))) {
SERVER_LOG(WARN, "fail to get store range", K(ret), K(macro_desc.range_));
}
} else if (OB_FAIL(macro_desc.range_.to_store_range(cols_desc_,
rowkey_allocator_,
info.store_range_))) {
@ -626,14 +634,9 @@ int ObAllVirtualTabletSSTableMacroInfo::get_next_sstable()
SERVER_LOG(WARN, "fail to get curr sstable meta handle", K(ret));
} else {
const storage::ObITableReadInfo *index_read_info = nullptr;
if (curr_sstable_->is_normal_cg_sstable()) {
if (OB_FAIL(MTL(ObTenantCGReadInfoMgr *)->get_index_read_info(index_read_info))) {
SERVER_LOG(WARN, "failed to get index read info from ObTenantCGReadInfoMgr", KR(ret));
}
} else {
index_read_info = &tablet_handle_.get_obj()->get_rowkey_read_info();
}
if (FAILEDx(curr_sstable_->scan_macro_block(
if (OB_FAIL(tablet_handle_.get_obj()->get_sstable_read_info(curr_sstable_, index_read_info))) {
SERVER_LOG(WARN, "failed to get index read info ", KR(ret), KPC_(curr_sstable));
} else if (OB_FAIL(curr_sstable_->scan_macro_block(
curr_range_,
*index_read_info,
iter_allocator_,

13
src/observer/virtual_table/ob_mds_event_buffer.h
View File

@ -23,6 +23,7 @@
#include "storage/multi_data_source/runtime_utility/common_define.h"
#include "util/easy_time.h"
#include "share/ob_task_define.h"
#include "storage/tx/ob_tx_seq.h"
namespace oceanbase
{
@ -85,7 +86,7 @@ struct MdsEvent {
unit_id_(UINT8_MAX),
writer_type_(storage::mds::WriterType::UNKNOWN_WRITER),
writer_id_(0),
seq_no_(0),
seq_no_(),
redo_scn_(),
end_scn_(),
trans_version_(),
@ -131,7 +132,7 @@ private:
uint8_t unit_id,
storage::mds::WriterType writer_type,
int64_t writer_id,
int64_t seq_no,
transaction::ObTxSEQ seq_no,
share::SCN redo_scn,
share::SCN end_scn,
share::SCN trans_version,
@ -213,7 +214,7 @@ private:
uint8_t unit_id_;
storage::mds::WriterType writer_type_;
int64_t writer_id_;
int64_t seq_no_;
transaction::ObTxSEQ seq_no_;
share::SCN redo_scn_;
share::SCN end_scn_;
share::SCN trans_version_;
@ -255,9 +256,9 @@ struct ObMdsEventBuffer {
}
void append(const MdsEventKey &key, const MdsEvent &event, const char *file, const uint32_t line, const char *func) {
if (OB_NOT_NULL(file) && OB_UNLIKELY(line != 0) && OB_NOT_NULL(func) && OB_NOT_NULL(event.event_)) {
//share::ObTaskController::get().allow_next_syslog();
//::oceanbase::common::OB_PRINT("[MDS.EVENT]", OB_LOG_LEVEL_INFO, file, line, func, OB_LOG_LOCATION_HASH_VAL, OB_SUCCESS,
//event.event_, LOG_KVS(K(key), K(event)));
// share::ObTaskController::get().allow_next_syslog();
// ::oceanbase::common::OB_PRINT("[MDS.EVENT]", OB_LOG_LEVEL_INFO, file, line, func, OB_LOG_LOCATION_HASH_VAL, OB_SUCCESS,
// event.event_, LOG_KVS(K(key), K(event)));
}
if (is_inited_) {
(void) mds_event_cache_.append(key, event, file, line, func);

21
src/rootserver/ob_tenant_transfer_service.cpp
View File

@ -25,6 +25,7 @@
#include "share/ob_balance_define.h" // ObBalanceTaskID
#include "share/location_cache/ob_location_service.h" // location_service_
#include "share/ob_srv_rpc_proxy.h" // srv_rpc_proxy_
#include "share/ob_share_util.h" // ObShareUtil
#include "storage/ob_common_id_utils.h" // ObCommonIDUtils
#include "storage/tablelock/ob_table_lock_service.h" // ObTableLockService
#include "observer/ob_inner_sql_connection.h" // ObInnerSQLConnection
@ -1142,9 +1143,9 @@ int ObTenantTransferService::generate_transfer_task(
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
if (IS_NOT_INIT || OB_ISNULL(sql_proxy_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret));
LOG_WARN("not init", KR(ret), KP(sql_proxy_));
} else if (OB_UNLIKELY(!src_ls.is_valid()
|| !dest_ls.is_valid()
|| part_list.empty()
@ -1159,8 +1160,11 @@ int ObTenantTransferService::generate_transfer_task(
ObTransferStatus status(ObTransferStatus::INIT);
ObTransferPartList transfer_part_list;
const int64_t part_count = min(get_tablet_count_threshold_(), part_list.count());
uint64_t data_version = 0;
if (OB_FAIL(transfer_part_list.reserve(part_count))) {
LOG_WARN("reserve failed", KR(ret), K(part_count));
} else if (OB_FAIL(ObShareUtil::fetch_current_data_version(*sql_proxy_, tenant_id_, data_version))) { // can not use trans
LOG_WARN("fetch current data version failed", KR(ret), K(tenant_id_));
} else if (OB_FAIL(ObTransferTaskOperator::generate_transfer_task_id(trans, tenant_id_, task_id))) {
LOG_WARN("fail to generate transfer task id", KR(ret), K_(tenant_id));
} else {
@ -1170,9 +1174,18 @@ int ObTenantTransferService::generate_transfer_task(
LOG_WARN("push back failed", KR(ret), K(i), K(part_list), K(transfer_part_list));
}
}
if (FAILEDx(task.init(task_id, src_ls, dest_ls, transfer_part_list, status, trace_id, balance_task_id))) {
if (FAILEDx(task.init(
task_id,
src_ls,
dest_ls,
transfer_part_list,
status,
trace_id,
balance_task_id,
data_version))) {
LOG_WARN("init transfer task failed", KR(ret), K(task_id), K(src_ls),
K(dest_ls), K(transfer_part_list), K(status), K(trace_id), K(balance_task_id));
K(dest_ls), K(transfer_part_list), K(status), K(trace_id),
K(balance_task_id), K(data_version));
} else if (OB_FAIL(ObTransferTaskOperator::insert(trans, tenant_id_, task))) {
LOG_WARN("insert failed", KR(ret), K_(tenant_id), K(task));
}

4
src/share/backup/ob_backup_struct.h
View File

@ -1093,6 +1093,10 @@ struct ObBackupDataType final
{
OB_UNIS_VERSION(1);
public:
// TODO(yanfeng): change this comment when quick_restore branch merge
// backup sys: ls inner tablet, the granularity of success is log stream level
// backup minor: mini/minor/ddl/mds sstable, the granularity of success is tablet level
// backup major: major sstable, the granularity of success is macro block level
enum BackupDataType
{
BACKUP_SYS = 0,

8
src/share/ls/ob_ls_operator.cpp
View File

@ -495,9 +495,11 @@ int ObLSAttrOperator::process_sub_trans_(const ObLSAttr &ls_attr, ObMySQLTransac
} else if (OB_UNLIKELY(pos > length)) {
ret = OB_SIZE_OVERFLOW;
LOG_WARN("serialize error", KR(ret), K(pos), K(length));
} else if (OB_FAIL(txs->register_mds_into_tx(
*trans_desc, SYS_LS,
transaction::ObTxDataSourceType::LS_TABLE, buf, length))) {
} else if (OB_FAIL(txs->register_mds_into_tx(*trans_desc,
SYS_LS,
transaction::ObTxDataSourceType::LS_TABLE,
buf,
length))) {
LOG_WARN("failed to register tx data", KR(ret), KPC(trans_desc), K(expire_ts));
}
LOG_INFO("process sub trans", KR(ret), K(ls_attr));

28
src/share/ob_rpc_struct.cpp Executable file → Normal file
View File

@ -10184,13 +10184,26 @@ DEF_TO_STRING(ObCheckLSCanOfflineArg)
OB_SERIALIZE_MEMBER(ObCheckLSCanOfflineArg, tenant_id_, id_, current_ls_status_);
ObRegisterTxDataArg::ObRegisterTxDataArg()
: tenant_id_(OB_INVALID_TENANT_ID),
tx_desc_(nullptr),
ls_id_(),
type_(transaction::ObTxDataSourceType::UNKNOWN),
buf_(),
seq_no_(),
request_id_(0),
register_flag_()
{
}
bool ObRegisterTxDataArg::is_valid() const
{
return tenant_id_ != OB_INVALID_TENANT_ID &&
OB_NOT_NULL(tx_desc_) &&
tx_desc_->is_valid() &&
ls_id_.is_valid() &&
type_ != ObTxDataSourceType::UNKNOWN;
type_ != ObTxDataSourceType::UNKNOWN &&
seq_no_.is_valid();
}
int ObRegisterTxDataArg::init(const uint64_t tenant_id,
@ -10198,20 +10211,22 @@ int ObRegisterTxDataArg::init(const uint64_t tenant_id,
const ObLSID &ls_id,
const ObTxDataSourceType &type,
const ObString &buf,
const transaction::ObTxSEQ seq_no,
const int64_t base_request_id,
const transaction::ObRegisterMdsFlag &register_flag)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(tenant_id == OB_INVALID_TENANT_ID || !tx_desc.is_valid() || !ls_id.is_valid()
|| type == ObTxDataSourceType::UNKNOWN)) {
|| type == ObTxDataSourceType::UNKNOWN || !seq_no.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(tx_desc), K(ls_id), K(type));
LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(tx_desc), K(ls_id), K(type), K(seq_no));
} else {
tenant_id_ = tenant_id;
tx_desc_ = const_cast<ObTxDesc *>(&tx_desc);
ls_id_ = ls_id;
type_ = type;
buf_ = buf;
seq_no_ = seq_no;
request_id_ = base_request_id;
register_flag_ = register_flag;
}
@ -10225,6 +10240,7 @@ void ObRegisterTxDataArg::reset()
ls_id_.reset();
type_ = ObTxDataSourceType::UNKNOWN;
buf_.reset();
seq_no_.reset();
request_id_ = 0;
register_flag_.reset();
return;
@ -10244,7 +10260,7 @@ OB_DEF_SERIALIZE(ObRegisterTxDataArg)
int ret = OB_SUCCESS;
OB_UNIS_ENCODE(tenant_id_);
OB_UNIS_ENCODE(*tx_desc_);
LST_DO_CODE(OB_UNIS_ENCODE, ls_id_, type_, buf_, request_id_, register_flag_);
LST_DO_CODE(OB_UNIS_ENCODE, ls_id_, type_, buf_, request_id_, register_flag_, seq_no_);
return ret;
}
OB_DEF_DESERIALIZE(ObRegisterTxDataArg)
@ -10259,7 +10275,7 @@ OB_DEF_DESERIALIZE(ObRegisterTxDataArg)
} else if (OB_FAIL(tx_svc->acquire_tx(buf, data_len, pos, tx_desc_))) {
LOG_WARN("acquire tx by deserialize fail", K(data_len), K(pos), KR(ret));
} else {
LST_DO_CODE(OB_UNIS_DECODE, ls_id_, type_, buf_, request_id_, register_flag_);
LST_DO_CODE(OB_UNIS_DECODE, ls_id_, type_, buf_, request_id_, register_flag_, seq_no_);
LOG_INFO("deserialize txDesc from session", KPC_(tx_desc), KPC(this));
}
}
@ -10270,7 +10286,7 @@ OB_DEF_SERIALIZE_SIZE(ObRegisterTxDataArg)
int64_t len = 0;
OB_UNIS_ADD_LEN(tenant_id_);
OB_UNIS_ADD_LEN(*tx_desc_);
LST_DO_CODE(OB_UNIS_ADD_LEN, ls_id_, type_, buf_, request_id_, register_flag_);
LST_DO_CODE(OB_UNIS_ADD_LEN, ls_id_, type_, buf_, request_id_, register_flag_, seq_no_);
return len;
}

11
src/share/ob_rpc_struct.h Executable file → Normal file
View File

@ -58,6 +58,7 @@
#include "storage/blocksstable/ob_block_sstable_struct.h"
#include "storage/ddl/ob_ddl_struct.h"
#include "storage/tx/ob_trans_define.h"
#include "storage/tx/ob_multi_data_source.h"
#include "share/unit/ob_unit_info.h" //ObUnit*
#include "share/backup/ob_backup_clean_struct.h"
#include "logservice/palf/palf_options.h"//access mode
@ -10306,11 +10307,8 @@ struct ObRegisterTxDataArg
OB_UNIS_VERSION(1);
public:
ObRegisterTxDataArg()
: tenant_id_(OB_INVALID_TENANT_ID), tx_desc_(nullptr), ls_id_(),
type_(transaction::ObTxDataSourceType::UNKNOWN), buf_(), request_id_(0),register_flag_()
{}
~ObRegisterTxDataArg() {}
ObRegisterTxDataArg();
~ObRegisterTxDataArg() = default;
bool is_valid() const;
void reset();
void inc_request_id(const int64_t base_request_id);
@ -10319,6 +10317,7 @@ public:
const share::ObLSID &ls_id,
const transaction::ObTxDataSourceType &type,
const common::ObString &buf,
const transaction::ObTxSEQ seq_no,
const int64_t base_request_id,
const transaction::ObRegisterMdsFlag &register_flag);
TO_STRING_KV(K_(tenant_id),
@ -10326,6 +10325,7 @@ public:
K_(ls_id),
K_(type),
KP(buf_.length()),
K_(seq_no),
K_(request_id),
K_(register_flag));
@ -10335,6 +10335,7 @@ public:
share::ObLSID ls_id_;
transaction::ObTxDataSourceType type_;
common::ObString buf_;
transaction::ObTxSEQ seq_no_;
int64_t request_id_;
transaction::ObRegisterMdsFlag register_flag_;

0
src/share/ob_srv_rpc_proxy.h Executable file → Normal file
View File

3
src/share/parameter/ob_parameter_seed.ipp
View File

@ -1650,6 +1650,9 @@ ERRSIM_DEF_INT(errsim_max_ddl_block_count, OB_CLUSTER_PARAMETER, "0", "[0,)",
ERRSIM_DEF_STR(errsim_migration_src_server_addr, OB_CLUSTER_PARAMETER, "",
"the server dest ls choose as src when in errsim mode",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
ERRSIM_DEF_STR(errsim_migration_dest_server_addr, OB_CLUSTER_PARAMETER, "",
"the migration dest in errsim mode",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
DEF_BOOL(enable_cgroup, OB_CLUSTER_PARAMETER, "True",
"when set to false, cgroup will not init; when set to true but cgroup root dir is not ready, print ERROR",

6
src/share/scheduler/ob_tenant_dag_scheduler.h
View File

@ -217,6 +217,7 @@ public:
TASK_TYPE_DDL_SPLIT_PREPARE = 61,
TASK_TYPE_DDL_SPLIT_WRITE = 62,
TASK_TYPE_DDL_SPLIT_MERGE = 63,
TASK_TYPE_TABLE_FINISH_BACKFILL = 64,
TASK_TYPE_MAX,
};
@ -1435,6 +1436,11 @@ inline bool is_compaction_dag(ObDagType::ObDagTypeEnum dag_type)
ObDagType::DAG_TYPE_BATCH_FREEZE_TABLETS == dag_type;
}
inline bool is_ha_backfill_dag(const ObDagType::ObDagTypeEnum dag_type)
{
return ObDagType::DAG_TYPE_TABLET_BACKFILL_TX == dag_type;
}
inline int dag_yield()
{
int ret = OB_SUCCESS;

42
src/share/schema/ob_schema_utils.cpp
View File

@ -1213,6 +1213,48 @@ int ObSchemaUtils::batch_get_table_schemas_from_inner_table_(
return ret;
}
int ObSchemaUtils::check_whether_column_exist(
const uint64_t tenant_id,
const ObObjectID &table_id,
const ObString &column_name,
bool &exist)
{
int ret = OB_SUCCESS;
exist = false;
if (OB_ISNULL(GCTX.sql_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("GCTX.sql_proxy_ is null", KR(ret));
} else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id)
|| OB_INVALID_ID == table_id
|| column_name.empty()
|| !is_sys_table(table_id)
|| is_core_table(table_id))) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(tenant_id), K(table_id), K(column_name));
} else {
SMART_VAR(ObISQLClient::ReadResult, result) {
ObSqlString sql;
common::sqlclient::ObMySQLResult *res = NULL;
// in __all_column, tenant_id is primary key and it's value is 0
if (OB_FAIL(sql.append_fmt(
"SELECT count(*) = 1 AS exist FROM %s WHERE tenant_id = 0 and table_id = %lu and column_name = '%.*s'",
OB_ALL_COLUMN_TNAME, table_id, column_name.length(), column_name.ptr()))) {
LOG_WARN("fail to assign sql", KR(ret));
} else if (OB_FAIL(GCTX.sql_proxy_->read(result, tenant_id, sql.ptr()))) {
LOG_WARN("execute sql failed", KR(ret), K(sql));
} else if (OB_ISNULL(res = result.get_result())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get mysql result failed", KR(ret), K(tenant_id), K(sql));
} else if (OB_FAIL(res->next())) {
LOG_WARN("next failed", KR(ret), K(sql));
} else if (OB_FAIL(res->get_bool("exist", exist))) {
LOG_WARN("get max task id failed", KR(ret), K(sql));
}
}
}
return ret;
}
int ObSchemaUtils::is_drop_column_only(const AlterTableSchema &alter_table_schema, bool &is_drop_col_only)
{
int ret = OB_SUCCESS;

15
src/share/schema/ob_schema_utils.h
View File

@ -198,6 +198,21 @@ public:
const int64_t schema_version,
const bool skip_consensus);
// Use to check if the column of sys table (exclude core table) does exist
// by querying __all_column when the column is not accessible.
// (attention: the func contains an inner sql)
//
// @param[in] tenant_id: target tenant_id
// @param[in] table_id: sys table_id (exclude core table)
// @param[in] column_name: target column name
// @param[out] exist: whether the column really exists
// @return: OB_SUCCESS if success
static int check_whether_column_exist(
const uint64_t tenant_id,
const ObObjectID &table_id,
const ObString &column_name,
bool &exist);
// Use to check if the sys table (exclude core table) does exist
// by querying __all_table when the table is not accessible.
//

7
src/share/scn.h
View File

@ -12,6 +12,7 @@
#ifndef OCEABASE_SHARE_SCN_
#define OCEABASE_SHARE_SCN_
#include <iostream>
#include "lib/ob_define.h" // Serialization
#include "lib/utility/ob_print_utils.h" // Print*
namespace oceanbase {
@ -120,6 +121,12 @@ public:
int64_t get_serialize_size(void) const;
int to_yson(char *buf, const int64_t buf_len, int64_t &pos) const;
friend std::ostream &operator<<(std::ostream &os, const SCN &scn)
{
os << "ts_ns: " << scn.ts_ns_ << ", v: " << scn.v_;
return os;
}
TO_STRING_KV(K_(val), K_(v));
private:
void transform_max_();

22
src/share/transfer/ob_transfer_info.cpp
View File

@ -492,7 +492,8 @@ ObTransferTask::ObTransferTask()
result_(-1),
comment_(ObTransferTaskComment::EMPTY_COMMENT),
balance_task_id_(),
table_lock_owner_id_()
table_lock_owner_id_(),
data_version_(0)
{
}
@ -515,6 +516,7 @@ void ObTransferTask::reset()
comment_ = ObTransferTaskComment::EMPTY_COMMENT;
balance_task_id_.reset();
table_lock_owner_id_.reset();
data_version_ = 0;
}
// init by necessary info, other members take default values
@ -525,7 +527,8 @@ int ObTransferTask::init(
const ObTransferPartList &part_list,
const ObTransferStatus &status,
const common::ObCurTraceId::TraceId &trace_id,
const ObBalanceTaskID balance_task_id)
const ObBalanceTaskID balance_task_id,
const uint64_t data_version)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!task_id.is_valid()
@ -551,6 +554,7 @@ int ObTransferTask::init(
balance_task_id_ = balance_task_id;
start_scn_.set_min();
finish_scn_.set_min();
data_version_ = data_version;
}
}
return ret;
@ -573,7 +577,8 @@ int ObTransferTask::init(
const int result,
const ObTransferTaskComment &comment,
const ObBalanceTaskID balance_task_id,
const transaction::tablelock::ObTableLockOwnerID &lock_owner_id)
const transaction::tablelock::ObTableLockOwnerID &lock_owner_id,
const uint64_t data_version)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!task_id.is_valid()
@ -609,6 +614,7 @@ int ObTransferTask::init(
comment_ = comment;
balance_task_id_ = balance_task_id;
table_lock_owner_id_ = lock_owner_id;
data_version_ = data_version;
}
return ret;
}
@ -640,6 +646,7 @@ int ObTransferTask::assign(const ObTransferTask &other)
comment_ = other.comment_;
balance_task_id_ = other.balance_task_id_;
table_lock_owner_id_ = other.table_lock_owner_id_;
data_version_ = other.data_version_;
}
return ret;
}
@ -670,7 +677,8 @@ ObTransferTaskInfo::ObTransferTaskInfo()
tablet_list_(),
start_scn_(),
finish_scn_(),
result_(OB_SUCCESS)
result_(OB_SUCCESS),
data_version_(0)
{
}
@ -688,6 +696,7 @@ void ObTransferTaskInfo::reset()
start_scn_.reset();
finish_scn_.reset();
result_ = OB_SUCCESS;
data_version_ = 0;
}
// table_lock_tablet_list_ may be empty
@ -700,7 +709,8 @@ bool ObTransferTaskInfo::is_valid() const
&& !trace_id_.is_invalid()
&& status_.is_valid()
&& table_lock_owner_id_.is_valid()
&& !tablet_list_.empty();
&& !tablet_list_.empty()
&& data_version_ >= 0;
}
int ObTransferTaskInfo::convert_from(const uint64_t tenant_id, const ObTransferTask &task)
@ -724,6 +734,7 @@ int ObTransferTaskInfo::convert_from(const uint64_t tenant_id, const ObTransferT
start_scn_ = task.get_start_scn();
finish_scn_ = task.get_finish_scn();
result_ = task.get_result();
data_version_ = task.get_data_version();
}
return ret;
}
@ -749,6 +760,7 @@ int ObTransferTaskInfo::assign(const ObTransferTaskInfo &task_info)
start_scn_ = task_info.start_scn_;
finish_scn_ = task_info.finish_scn_;
result_ = task_info.result_;
data_version_ = task_info.data_version_;
}
return ret;
}

13
src/share/transfer/ob_transfer_info.h
View File

@ -324,7 +324,8 @@ public:
const ObTransferPartList &part_list,
const ObTransferStatus &status,
const common::ObCurTraceId::TraceId &trace_id,
const ObBalanceTaskID balance_task_id);
const ObBalanceTaskID balance_task_id,
const uint64_t data_version);
// init all members
int init(
@ -343,7 +344,8 @@ public:
const int result,
const ObTransferTaskComment &comment,
const ObBalanceTaskID balance_task_id,
const transaction::tablelock::ObTableLockOwnerID &lock_owner_id);
const transaction::tablelock::ObTableLockOwnerID &lock_owner_id,
const uint64_t data_version);
int assign(const ObTransferTask &other);
bool is_valid() const;
@ -374,10 +376,11 @@ public:
{
return table_lock_owner_id_;
}
uint64_t get_data_version() const { return data_version_; }
TO_STRING_KV(K_(task_id), K_(src_ls), K_(dest_ls), K_(part_list),
K_(not_exist_part_list), K_(lock_conflict_part_list), K_(table_lock_tablet_list), K_(tablet_list), K_(start_scn), K_(finish_scn),
K_(status), K_(trace_id), K_(result), K_(comment), K_(balance_task_id), K_(table_lock_owner_id));
K_(status), K_(trace_id), K_(result), K_(comment), K_(balance_task_id), K_(table_lock_owner_id), K_(data_version));
private:
ObTransferTaskID task_id_;
@ -396,6 +399,7 @@ private:
ObTransferTaskComment comment_;
ObBalanceTaskID balance_task_id_;
transaction::tablelock::ObTableLockOwnerID table_lock_owner_id_;
uint64_t data_version_; // for upgrade compatibility
};
struct ObTransferTaskInfo final
@ -409,7 +413,7 @@ struct ObTransferTaskInfo final
int fill_tablet_ids(ObIArray<ObTabletID> &tablet_ids) const;
TO_STRING_KV(K_(tenant_id), K_(src_ls_id), K_(dest_ls_id), K_(task_id), K_(trace_id),
K_(status), K_(table_lock_owner_id), K_(table_lock_tablet_list), K_(tablet_list),
K_(start_scn), K_(finish_scn), K_(result));
K_(start_scn), K_(finish_scn), K_(result), K_(data_version));
uint64_t tenant_id_;
share::ObLSID src_ls_id_;
@ -423,6 +427,7 @@ struct ObTransferTaskInfo final
share::SCN start_scn_;
share::SCN finish_scn_;
int32_t result_;
uint64_t data_version_; // for upgrade compatibility
DISALLOW_COPY_AND_ASSIGN(ObTransferTaskInfo);
};

107
src/share/transfer/ob_transfer_task_operator.cpp
View File

@ -50,7 +50,7 @@ int ObTransferTaskOperator::get(
} else if (OB_ISNULL(result.get_result())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get mysql result failed", KR(ret), K(tenant_id), K(sql));
} else if (OB_FAIL(construct_transfer_task_(*result.get_result(), task))) {
} else if (OB_FAIL(construct_transfer_task_(tenant_id, *result.get_result(), task))) {
LOG_WARN("construct transfer task failed", KR(ret), K(tenant_id), K(task_id), K(sql), K(task));
}
}
@ -91,7 +91,7 @@ int ObTransferTaskOperator::get_task_with_time(
} else {
LOG_WARN("get next result failed", KR(ret), K(sql), K(tenant_id), K(task_id));
}
} else if (OB_FAIL(parse_sql_result_(*res, with_time, task, create_time, finish_time))) {
} else if (OB_FAIL(parse_sql_result_(tenant_id, *res, with_time, task, create_time, finish_time))) {
LOG_WARN("parse sql result failed", KR(ret), K(tenant_id), K(task_id), K(task));
} else if (OB_FAIL(res->next())) {
if (OB_ITER_END == ret) {
@ -129,7 +129,7 @@ int ObTransferTaskOperator::get_by_status(
} else if (OB_ISNULL(result.get_result())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get mysql result failed", KR(ret), K(tenant_id), K(sql));
} else if (OB_FAIL(construct_transfer_tasks_(*result.get_result(), tasks))) {
} else if (OB_FAIL(construct_transfer_tasks_(tenant_id, *result.get_result(), tasks))) {
LOG_WARN("construct transfer task failed", KR(ret), K(tenant_id), K(sql), K(tasks));
} else if (tasks.empty()) {
ret = OB_ENTRY_NOT_EXIST;
@ -266,6 +266,24 @@ int ObTransferTaskOperator::fill_dml_splicer_(
|| OB_FAIL(dml_splicer.add_column("table_lock_owner_id", task.get_table_lock_owner_id().raw_value()))) {
LOG_WARN("fail to add column", KR(ret), K(task));
}
if (OB_FAIL(ret)) {
} else if (task.get_data_version() < MOCK_DATA_VERSION_4_2_3_0
|| (task.get_data_version() >= DATA_VERSION_4_3_0_0 && task.get_data_version() < DATA_VERSION_4_3_2_0)) {
// do nothing
} else {
char version_buf[common::OB_CLUSTER_VERSION_LENGTH] = {'\0'};
int64_t len = ObClusterVersion::print_version_str(
version_buf,
common::OB_CLUSTER_VERSION_LENGTH,
task.get_data_version());
if (OB_FAIL(ret)) {
} else if (OB_UNLIKELY(len < 0)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid version", KR(ret), K(task));
} else if (OB_FAIL(dml_splicer.add_column("data_version", version_buf))) {
LOG_WARN("add column failed", KR(ret), K(task));
}
}
return ret;
}
@ -726,7 +744,7 @@ int ObTransferTaskOperator::get_by_ls_id_(
} else if (OB_ISNULL(result.get_result())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get mysql result failed", KR(ret), K(tenant_id), K(sql));
} else if (OB_FAIL(construct_transfer_task_(*result.get_result(), task))) {
} else if (OB_FAIL(construct_transfer_task_(tenant_id, *result.get_result(), task))) {
if (OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("construct transfer task failed", KR(ret), K(tenant_id), K(ls_id), K(sql));
} else {
@ -740,6 +758,7 @@ int ObTransferTaskOperator::get_by_ls_id_(
}
int ObTransferTaskOperator::construct_transfer_tasks_(
const uint64_t tenant_id,
common::sqlclient::ObMySQLResult &res,
ObIArray<ObTransferTask> &tasks)
{
@ -758,8 +777,8 @@ int ObTransferTaskOperator::construct_transfer_tasks_(
} else {
LOG_WARN("get next result failed", KR(ret));
}
} else if (OB_FAIL(parse_sql_result_(res, with_time, task, create_time, finish_time))) {
LOG_WARN("parse sql result failed", KR(ret), K(task));
} else if (OB_FAIL(parse_sql_result_(tenant_id, res, with_time, task, create_time, finish_time))) {
LOG_WARN("parse sql result failed", KR(ret), K(tenant_id), K(task));
} else if (OB_FAIL(tasks.push_back(task))) {
LOG_WARN("fail to push back", KR(ret), K(task), K(tasks));
}
@ -768,6 +787,7 @@ int ObTransferTaskOperator::construct_transfer_tasks_(
}
int ObTransferTaskOperator::construct_transfer_task_(
const uint64_t tenant_id,
common::sqlclient::ObMySQLResult &res,
ObTransferTask &task)
{
@ -781,8 +801,8 @@ int ObTransferTaskOperator::construct_transfer_task_(
} else {
LOG_WARN("get next result failed", KR(ret));
}
} else if (OB_FAIL(parse_sql_result_(res, with_time, task, create_time, finish_time))) {
LOG_WARN("parse sql result failed", KR(ret), K(task));
} else if (OB_FAIL(parse_sql_result_(tenant_id, res, with_time, task, create_time, finish_time))) {
LOG_WARN("parse sql result failed", KR(ret), K(tenant_id), K(task));
} else if (OB_FAIL(res.next())) {
if (OB_ITER_END == ret) {
ret = OB_SUCCESS;
@ -797,6 +817,7 @@ int ObTransferTaskOperator::construct_transfer_task_(
}
int ObTransferTaskOperator::parse_sql_result_(
const uint64_t tenant_id,
common::sqlclient::ObMySQLResult &res,
const bool with_time,
ObTransferTask &task,
@ -829,6 +850,10 @@ int ObTransferTaskOperator::parse_sql_result_(
SCN finish_scn;
ObTableLockOwnerID owner_id;
common::ObCurTraceId::TraceId trace_id;
uint64_t data_version = 0;
ObString data_version_str;
bool data_version_is_null = false;
bool data_version_not_exist = false;
if (with_time) {
(void)GET_COL_IGNORE_NULL(res.get_int, "create_time_int64", create_time);
@ -850,6 +875,21 @@ int ObTransferTaskOperator::parse_sql_result_(
(void)GET_COL_IGNORE_NULL(res.get_int, "balance_task_id", balance_task_id);
(void)GET_COL_IGNORE_NULL(res.get_int, "table_lock_owner_id", lock_owner_val);
EXTRACT_STRBUF_FIELD_MYSQL(res, "trace_id", trace_id_buf, OB_MAX_TRACE_ID_BUFFER_SIZE, real_length);
EXTRACT_VARCHAR_FIELD_MYSQL_SKIP_RET_WITH_COLUMN_INFO(res, "data_version", data_version_str,
data_version_is_null, data_version_not_exist);
if (OB_FAIL(ret)) {
} else if (OB_UNLIKELY(data_version_is_null)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("data_version can not be null", KR(ret), K(data_version_is_null));
} else if (OB_FAIL(convert_data_version_(
tenant_id,
with_time,
data_version_not_exist,
data_version_str,
data_version))) {
LOG_WARN("convert data_version failed", KR(ret), K(tenant_id),
K(with_time), K(data_version_not_exist), K(data_version_str));
}
if (OB_FAIL(ret)) {
} else if (start_scn_val != OB_INVALID_SCN_VAL
@ -880,10 +920,11 @@ int ObTransferTaskOperator::parse_sql_result_(
static_cast<int32_t>(result),
str_to_transfer_task_comment(comment),
ObBalanceTaskID(balance_task_id),
owner_id))) {
owner_id,
data_version))) {
LOG_WARN("fail to init transfer task", KR(ret), K(task_id), K(src_ls), K(dest_ls), K(part_list_str),
K(not_exist_part_list_str), K(lock_conflict_part_list_str), K(table_lock_tablet_list_str), K(tablet_list_str), K(start_scn),
K(finish_scn), K(status), K(trace_id), K(result), K(comment), K(balance_task_id), K(owner_id));
K(finish_scn), K(status), K(trace_id), K(result), K(comment), K(balance_task_id), K(owner_id), K(data_version));
}
return ret;
@ -1000,8 +1041,8 @@ int ObTransferTaskOperator::get_history_task(
} else {
LOG_WARN("next failed", KR(ret), K(sql));
}
} else if (OB_FAIL(parse_sql_result_(*res, with_time, task, create_time, finish_time))) {
LOG_WARN("parse sql result failed", KR(ret));
} else if (OB_FAIL(parse_sql_result_(tenant_id, *res, with_time, task, create_time, finish_time))) {
LOG_WARN("parse sql result failed", KR(ret), K(tenant_id));
} else {
LOG_TRACE("get history task status success", KR(ret), K(tenant_id), K(task_id), K(task));
}
@ -1089,8 +1130,8 @@ int ObTransferTaskOperator::get_last_task_by_balance_task_id(
} else {
LOG_WARN("next failed", KR(ret), K(sql));
}
} else if (OB_FAIL(parse_sql_result_(*res, with_time, last_task, create_time, finish_time))) {
LOG_WARN("parse sql result failed", KR(ret), K(with_time), K(sql));
} else if (OB_FAIL(parse_sql_result_(tenant_id, *res, with_time, last_task, create_time, finish_time))) {
LOG_WARN("parse sql result failed", KR(ret), K(tenant_id), K(with_time), K(sql));
} else {
LOG_TRACE("get last task by balance task id from history", KR(ret),
K(tenant_id), K(balance_task_id), K(last_task), K(create_time), K(finish_time));
@ -1451,5 +1492,43 @@ int ObTransferTaskOperator::batch_get_tablet_ls_cache(
return ret;
}
int ObTransferTaskOperator::convert_data_version_(
const uint64_t tenant_id,
const bool is_history,
const bool column_not_exist,
const ObString &data_version_str,
uint64_t &data_version)
{
int ret = OB_SUCCESS;
data_version = 0;
if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(tenant_id), K(is_history), K(data_version_str));
} else if (column_not_exist) {
bool column_exist_double_check = false;
const ObString column_name("data_version");
const uint64_t table_id = is_history ? OB_ALL_TRANSFER_TASK_HISTORY_TID : OB_ALL_TRANSFER_TASK_TID;
if (OB_FAIL(ObSchemaUtils::check_whether_column_exist(
tenant_id,
table_id,
column_name,
column_exist_double_check))) { // contain an inner sql
LOG_WARN("check column exist failed", KR(ret), K(tenant_id), K(table_id), K(column_name));
} else if (!column_exist_double_check) {
// 1. tenant data_version is old, column does not exist
data_version = DEFAULT_MIN_DATA_VERSION;
} else { // 2. column exists but schema is old
ret = OB_NEED_RETRY;
LOG_WARN("schema is old, can not read column data_version",
KR(ret), K(tenant_id), K(table_id), K(column_name));
}
} else if (data_version_str.empty()) {
data_version = DEFAULT_MIN_DATA_VERSION;
} else if (OB_FAIL(ObClusterVersion::get_version(data_version_str, data_version))) {
LOG_WARN("get version failed", KR(ret), K(data_version_str), K(data_version));
}
return ret;
}
} // end namespace share
} // end namespace oceanbase

9
src/share/transfer/ob_transfer_task_operator.h
View File

@ -483,12 +483,15 @@ private:
const int32_t group_id,
ObTransferTask &task);
static int construct_transfer_tasks_(
const uint64_t tenant_id,
common::sqlclient::ObMySQLResult &res,
common::ObIArray<ObTransferTask> &tasks);
static int construct_transfer_task_(
const uint64_t tenant_id,
common::sqlclient::ObMySQLResult &res,
ObTransferTask &task);
static int parse_sql_result_(
const uint64_t tenant_id,
common::sqlclient::ObMySQLResult &res,
const bool with_time,
ObTransferTask &task,
@ -501,6 +504,12 @@ private:
ObDMLSqlSplicer &dml_splicer,
common::ObArenaAllocator &allocator,
const ObTransferTask &task);
static int convert_data_version_(
const uint64_t tenant_id,
const bool is_history,
const bool column_not_exist,
const ObString &data_version_str,
uint64_t &data_version);
};
} // end namespace share

3
src/sql/das/ob_das_scan_op.cpp
View File

@ -266,6 +266,7 @@ int ObDASScanOp::init_scan_param()
scan_param_.table_scan_opt_ = scan_ctdef_->table_scan_opt_;
scan_param_.fb_snapshot_ = scan_rtdef_->fb_snapshot_;
scan_param_.fb_read_tx_uncommitted_ = scan_rtdef_->fb_read_tx_uncommitted_;
scan_param_.is_mds_query_ = false;
if (scan_rtdef_->is_for_foreign_check_) {
scan_param_.trans_desc_ = trans_desc_;
}
@ -310,7 +311,7 @@ int ObDASScanOp::init_scan_param()
scan_param_.external_file_format_.csv_format_.file_column_nums_ = static_cast<int64_t>(max_idx);
}
}
LOG_DEBUG("init scan param", K(scan_param_));
LOG_DEBUG("init scan param", K(ret), K(scan_param_));
return ret;
}

1
src/sql/engine/px/datahub/components/ob_dh_range_dist_wf.cpp
View File

@ -12,6 +12,7 @@
#define USING_LOG_PREFIX SQL_ENG
#include "lib/utility/ob_sort.h"
#include "sql/engine/px/datahub/components/ob_dh_range_dist_wf.h"
#include "sql/engine/px/ob_px_util.h"
#include "sql/engine/window_function/ob_window_function_op.h"

3
src/sql/engine/window_function/ob_window_function_vec_op.cpp
View File

@ -12,6 +12,7 @@
#define USING_LOG_PREFIX SQL_ENG
#include "lib/utility/ob_print_utils.h"
#include "lib/utility/ob_sort.h"
#include "ob_window_function_vec_op.h"
#include "share/aggregate/iaggregate.h"
#include "share/aggregate/processor.h"
@ -4019,4 +4020,4 @@ int ObWindowFunctionVecOpInput::sync_wait(ObExecContext &ctx, ObReportingWFWhole
}
} // end sql
} // end oceanbase
} // end oceanbase

12
src/storage/CMakeLists.txt
View File

@ -247,6 +247,11 @@ ob_set_subtarget(ob_storage backup
ob_set_subtarget(ob_storage tablet
tablet/ob_batch_create_tablet_pretty_arg.cpp
tablet/ob_full_tablet_creator.cpp
tablet/ob_i_tablet_mds_interface.cpp
tablet/ob_mds_range_query_iterator.cpp
tablet/ob_mds_row_iterator.cpp
tablet/ob_mds_scan_param_helper.cpp
tablet/ob_mds_schema_helper.cpp
tablet/ob_tablet_binding_helper.cpp
tablet/ob_tablet_binding_replay_executor.cpp
tablet/ob_tablet_complex_addr.cpp
@ -278,6 +283,7 @@ ob_set_subtarget(ob_storage tablet
tablet/ob_tablet_binding_info.cpp
tablet/ob_tablet_binding_mds_user_data.cpp
tablet/ob_tablet_mds_node_dump_operator.cpp
tablet/ob_tablet_mds_table_mini_merger.cpp
tablet/ob_tablet_start_transfer_mds_helper.cpp
tablet/ob_tablet_finish_transfer_mds_helper.cpp
tablet/ob_tablet_persister.cpp
@ -397,6 +403,8 @@ ob_set_subtarget(ob_storage tx
tx/ob_xa_trans_heartbeat_worker.cpp
tx/ob_tx_ctx_mds.cpp
tx/ob_multi_data_source.cpp
tx/ob_multi_data_source_printer.cpp
tx/ob_multi_data_source_tx_buffer_node.cpp
tx/ob_tx_free_route.cpp
tx/ob_tx_free_route_state.cpp
tx/ob_tx_free_route_rpc.cpp
@ -690,16 +698,20 @@ ob_set_subtarget(ob_storage multi_data_source
multi_data_source/mds_table_base.cpp
multi_data_source/mds_writer.cpp
multi_data_source/mds_table_handler.cpp
multi_data_source/adapter_define/mds_dump_kv_wrapper.cpp
multi_data_source/adapter_define/mds_dump_node.cpp
multi_data_source/runtime_utility/common_define.cpp
multi_data_source/runtime_utility/mds_tenant_service.cpp
multi_data_source/runtime_utility/mds_factory.cpp
multi_data_source/runtime_utility/mds_retry_control.cpp
multi_data_source/ob_mds_table_merge_dag.cpp
multi_data_source/ob_mds_table_merge_dag_param.cpp
multi_data_source/ob_mds_table_merge_task.cpp
multi_data_source/ob_mds_compaction_filter.cpp
multi_data_source/ob_start_transfer_in_mds_ctx.cpp
multi_data_source/ob_finish_transfer_in_mds_ctx.cpp
multi_data_source/ob_tablet_create_mds_ctx.cpp
multi_data_source/ob_tablet_mds_merge_ctx.cpp
multi_data_source/test/example_user_helper_define.cpp
)

1
src/storage/access/ob_dml_param.cpp
View File

@ -297,6 +297,7 @@ DEF_TO_STRING(ObTableScanParam)
K_(sample_info),
K_(need_scn),
K_(need_switch_param),
K_(is_mds_query),
K_(fb_read_tx_uncommitted),
K_(external_file_format),
K_(external_file_location));

2
src/storage/access/ob_dml_param.h
View File

@ -136,6 +136,7 @@ public:
allocator_(&CURRENT_CONTEXT->get_arena_allocator()),
need_scn_(false),
need_switch_param_(false),
is_mds_query_(false),
is_thread_scope_(true)
{}
virtual ~ObTableScanParam() {}
@ -150,6 +151,7 @@ public:
common::SampleInfo sample_info_;
bool need_scn_;
bool need_switch_param_;
bool is_mds_query_;
OB_INLINE virtual bool is_valid() const {
return snapshot_.valid_ && ObVTableScanParam::is_valid();
}

64
src/storage/access/ob_multiple_merge.cpp
View File

@ -31,12 +31,15 @@
#include "storage/column_store/ob_column_oriented_sstable.h"
#include "storage/tablet/ob_tablet.h"
#include "storage/tx/ob_trans_part_ctx.h"
#include "storage/compaction/ob_tenant_tablet_scheduler.h"
#include "storage/concurrency_control/ob_data_validation_service.h"
namespace oceanbase
{
using namespace common;
using namespace blocksstable;
using namespace share::schema;
using namespace sql;
namespace storage
{
@ -778,6 +781,30 @@ void ObMultipleMerge::report_tablet_stat()
}
}
int ObMultipleMerge::update_and_report_tablet_stat()
{
int ret = OB_SUCCESS;
EVENT_ADD(ObStatEventIds::STORAGE_READ_ROW_COUNT, scan_cnt_);
if (NULL != access_ctx_->table_scan_stat_) {
access_ctx_->table_scan_stat_->access_row_cnt_ += access_ctx_->table_store_stat_.logical_read_cnt_;
access_ctx_->table_scan_stat_->rowkey_prefix_ = access_ctx_->table_store_stat_.rowkey_prefix_;
access_ctx_->table_scan_stat_->bf_filter_cnt_ += access_ctx_->table_store_stat_.bf_filter_cnt_;
access_ctx_->table_scan_stat_->bf_access_cnt_ += access_ctx_->table_store_stat_.bf_access_cnt_;
access_ctx_->table_scan_stat_->empty_read_cnt_ += access_ctx_->table_store_stat_.empty_read_cnt_;
access_ctx_->table_scan_stat_->fuse_row_cache_hit_cnt_ += access_ctx_->table_store_stat_.fuse_row_cache_hit_cnt_;
access_ctx_->table_scan_stat_->fuse_row_cache_miss_cnt_ += access_ctx_->table_store_stat_.fuse_row_cache_miss_cnt_;
access_ctx_->table_scan_stat_->block_cache_hit_cnt_ += access_ctx_->table_store_stat_.block_cache_hit_cnt_;
access_ctx_->table_scan_stat_->block_cache_miss_cnt_ += access_ctx_->table_store_stat_.block_cache_miss_cnt_;
access_ctx_->table_scan_stat_->row_cache_hit_cnt_ += access_ctx_->table_store_stat_.row_cache_hit_cnt_;
access_ctx_->table_scan_stat_->row_cache_miss_cnt_ += access_ctx_->table_store_stat_.row_cache_miss_cnt_;
}
if (MTL(compaction::ObTenantTabletScheduler *)->enable_adaptive_compaction()) {
report_tablet_stat();
}
access_ctx_->table_store_stat_.reuse();
return ret;
}
int ObMultipleMerge::process_fuse_row(const bool not_using_static_engine,
ObDatumRow &in_row,
ObDatumRow *&out_row)
@ -1243,10 +1270,19 @@ int ObMultipleMerge::prepare_read_tables(bool refresh)
{
int ret = OB_SUCCESS;
tables_.reset();
if (OB_UNLIKELY(OB_ISNULL(get_table_param_) || !access_param_->is_valid() || NULL == access_ctx_)) {
const bool is_mds_query = access_param_->iter_param_.is_mds_query_;
if (OB_UNLIKELY(NULL == get_table_param_ || !access_param_->is_valid() || NULL == access_ctx_)) {
ret = OB_NOT_INIT;
LOG_WARN("ObMultipleMerge has not been inited", K(ret), K_(get_table_param), KP_(access_param),
KP_(access_ctx));
} else if (is_mds_query) {
ObTableStoreIterator *table_store_iter = get_table_param_->tablet_iter_.table_iter();
table_store_iter->reset();
if (OB_FAIL(prepare_mds_tables(refresh))) {
LOG_WARN("fail to prepare mds tables", K(ret), K(refresh), K_(get_table_param), KPC_(access_param));
} else if (OB_FAIL(prepare_tables_from_iterator(*table_store_iter, &get_table_param_->sample_info_))) {
LOG_WARN("failed to prepare tables from iter", K(ret), KPC(table_store_iter));
}
} else if (!refresh && get_table_param_->tablet_iter_.table_iter()->is_valid()) {
if (OB_FAIL(prepare_tables_from_iterator(*get_table_param_->tablet_iter_.table_iter()))) {
LOG_WARN("prepare tables fail", K(ret), K(get_table_param_->tablet_iter_.table_iter()));
@ -1279,6 +1315,30 @@ int ObMultipleMerge::prepare_read_tables(bool refresh)
return ret;
}
int ObMultipleMerge::prepare_mds_tables(bool refresh)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(refresh)) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("mds query does not support refresh table currently", K(ret), K(refresh), K_(access_param));
} else {
ObTabletTableIterator &tablet_iter = get_table_param_->tablet_iter_;
int64_t snapshot_version = get_table_param_->frozen_version_;
if (-1 == snapshot_version) {
snapshot_version = access_ctx_->store_ctx_->mvcc_acc_ctx_.get_snapshot_version().get_val_for_tx();
}
if (OB_FAIL(tablet_iter.get_mds_sstables_from_tablet(snapshot_version))) {
LOG_WARN("fail to get mds sstables", K(ret), K_(get_table_param), K(snapshot_version));
} else {
LOG_DEBUG("succeed to get mds sstables from tablet", K(ret), K(tablet_iter));
}
}
return ret;
}
int ObMultipleMerge::prepare_tables_from_iterator(ObTableStoreIterator &table_iter, const common::SampleInfo *sample_info)
{
int ret = OB_SUCCESS;
@ -1370,7 +1430,7 @@ int ObMultipleMerge::refresh_table_on_demand()
} else if (FALSE_IT(reset_iter_array(access_param_->is_use_global_iter_pool()))) {
} else if (OB_FAIL(refresh_tablet_iter())) {
STORAGE_LOG(WARN, "fail to refresh tablet iter", K(ret));
} else if (OB_FAIL(prepare_read_tables(true))) {
} else if (OB_FAIL(prepare_read_tables(true/*refresh*/))) {
STORAGE_LOG(WARN, "fail to prepare read tables", K(ret));
} else if (OB_FAIL(reset_tables())) {
STORAGE_LOG(WARN, "fail to reset tables", K(ret));

35
src/storage/access/ob_multiple_merge.h
View File

@ -27,7 +27,6 @@
#include "ob_table_access_context.h"
#include "storage/meta_mem/ob_tablet_handle.h"
#include "storage/lob/ob_lob_data_reader.h"
#include "storage/compaction/ob_tenant_tablet_scheduler.h"
namespace oceanbase
{
@ -104,6 +103,7 @@ private:
// project to output expressions
int project2output_exprs(blocksstable::ObDatumRow &unprojected_row, blocksstable::ObDatumRow &cur_row);
int prepare_read_tables(bool refresh = false);
int prepare_mds_tables(bool refresh);
int prepare_tables_from_iterator(ObTableStoreIterator &table_iter, const common::SampleInfo *sample_info = nullptr);
int refresh_table_on_demand();
int refresh_tablet_iter();
@ -123,7 +123,7 @@ private:
int handle_lob_before_fuse_row();
void reuse_lob_locator();
void report_tablet_stat();
OB_INLINE int update_and_report_tablet_stat();
int update_and_report_tablet_stat();
void inner_reset();
protected:
@ -169,7 +169,12 @@ private:
OB_INLINE int ObMultipleMerge::check_need_refresh_table(bool &need_refresh)
{
int ret = OB_SUCCESS;
need_refresh = get_table_param_->tablet_iter_.table_iter()->check_store_expire();
if (access_param_->iter_param_.is_mds_query_) {
need_refresh = false;
} else {
need_refresh = get_table_param_->tablet_iter_.table_iter()->check_store_expire();
}
#ifdef ERRSIM
ret = OB_E(EventTable::EN_FORCE_REFRESH_TABLE) ret;
if (OB_FAIL(ret)) {
@ -180,30 +185,6 @@ OB_INLINE int ObMultipleMerge::check_need_refresh_table(bool &need_refresh)
return ret;
}
OB_INLINE int ObMultipleMerge::update_and_report_tablet_stat()
{
int ret = OB_SUCCESS;
EVENT_ADD(ObStatEventIds::STORAGE_READ_ROW_COUNT, scan_cnt_);
if (NULL != access_ctx_->table_scan_stat_) {
access_ctx_->table_scan_stat_->access_row_cnt_ += access_ctx_->table_store_stat_.logical_read_cnt_;
access_ctx_->table_scan_stat_->rowkey_prefix_ = access_ctx_->table_store_stat_.rowkey_prefix_;
access_ctx_->table_scan_stat_->bf_filter_cnt_ += access_ctx_->table_store_stat_.bf_filter_cnt_;
access_ctx_->table_scan_stat_->bf_access_cnt_ += access_ctx_->table_store_stat_.bf_access_cnt_;
access_ctx_->table_scan_stat_->empty_read_cnt_ += access_ctx_->table_store_stat_.empty_read_cnt_;
access_ctx_->table_scan_stat_->fuse_row_cache_hit_cnt_ += access_ctx_->table_store_stat_.fuse_row_cache_hit_cnt_;
access_ctx_->table_scan_stat_->fuse_row_cache_miss_cnt_ += access_ctx_->table_store_stat_.fuse_row_cache_miss_cnt_;
access_ctx_->table_scan_stat_->block_cache_hit_cnt_ += access_ctx_->table_store_stat_.block_cache_hit_cnt_;
access_ctx_->table_scan_stat_->block_cache_miss_cnt_ += access_ctx_->table_store_stat_.block_cache_miss_cnt_;
access_ctx_->table_scan_stat_->row_cache_hit_cnt_ += access_ctx_->table_store_stat_.row_cache_hit_cnt_;
access_ctx_->table_scan_stat_->row_cache_miss_cnt_ += access_ctx_->table_store_stat_.row_cache_miss_cnt_;
}
if (MTL(compaction::ObTenantTabletScheduler *)->enable_adaptive_compaction()) {
report_tablet_stat();
}
access_ctx_->table_store_stat_.reuse();
return ret;
}
}
}

2
src/storage/access/ob_single_merge.cpp
View File

@ -160,7 +160,7 @@ int ObSingleMerge::get_table_row(const int64_t table_idx,
if (prow->row_flag_.is_exist() && !has_uncommited_row) {
has_uncommited_row = prow->is_have_uncommited_row() || fuse_row.snapshot_version_ == INT64_MAX;
}
STORAGE_LOG(DEBUG, "process row fuse", KPC(prow), K(fuse_row), KPC(access_ctx_->store_ctx_));
STORAGE_LOG(DEBUG, "process row fuse", K(ret), KPC(prow), K(fuse_row), KPC(access_ctx_->store_ctx_));
}
}
return ret;

28
src/storage/access/ob_table_access_param.cpp
View File

@ -52,6 +52,7 @@ ObTableIterParam::ObTableIterParam()
has_lob_column_out_(false),
is_for_foreign_check_(false),
limit_prefetch_(false),
is_mds_query_(false),
is_non_unique_local_index_(false),
ss_rowkey_prefix_cnt_(0),
pd_storage_flag_(),
@ -100,6 +101,7 @@ void ObTableIterParam::reset()
has_lob_column_out_ = false;
is_for_foreign_check_ = false;
limit_prefetch_ = false;
is_mds_query_ = false;
is_non_unique_local_index_ = false;
table_scan_opt_.reset();
ObSSTableIndexFilterFactory::destroy_sstable_index_filter(sstable_index_filter_);
@ -185,6 +187,7 @@ DEF_TO_STRING(ObTableIterParam)
K_(has_lob_column_out),
K_(is_for_foreign_check),
K_(limit_prefetch),
K_(is_mds_query),
K_(is_non_unique_local_index),
K_(ss_rowkey_prefix_cnt),
K_(table_scan_opt));
@ -223,23 +226,35 @@ void ObTableAccessParam::reset()
int ObTableAccessParam::init(
const ObTableScanParam &scan_param,
const ObTabletHandle &tablet_handle)
const ObTabletHandle *tablet_handle,
const ObITableReadInfo *rowkey_read_info)
{
int ret = OB_SUCCESS;
const share::schema::ObTableParam &table_param = *scan_param.table_param_;
if(IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("ObTableAccessParam init twice", K(ret), K(*this));
} else if (OB_UNLIKELY(!tablet_handle.is_valid())) {
} else if (OB_ISNULL(scan_param.table_param_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("Invalid tablet handle", K(ret), K(tablet_handle));
LOG_WARN("invalid args", K(ret), KP(scan_param.table_param_));
} else if (OB_UNLIKELY(nullptr == rowkey_read_info && nullptr == tablet_handle)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", K(ret), KP(rowkey_read_info), KP(tablet_handle));
} else {
const share::schema::ObTableParam &table_param = *scan_param.table_param_;
iter_param_.table_id_ = table_param.get_table_id();
iter_param_.tablet_id_ = scan_param.tablet_id_;
iter_param_.read_info_ = &table_param.get_read_info();
if (nullptr == tablet_handle) {
iter_param_.rowkey_read_info_ = rowkey_read_info;
iter_param_.set_tablet_handle(nullptr);
} else {
iter_param_.rowkey_read_info_ = &tablet_handle->get_obj()->get_rowkey_read_info();
iter_param_.set_tablet_handle(tablet_handle);
}
iter_param_.cg_read_infos_ = table_param.get_cg_read_infos();
iter_param_.rowkey_read_info_ = &tablet_handle.get_obj()->get_rowkey_read_info();
iter_param_.set_tablet_handle(&tablet_handle);
iter_param_.out_cols_project_ = &table_param.get_output_projector();
iter_param_.agg_cols_project_ = &table_param.get_aggregate_projector();
iter_param_.group_by_cols_project_ = &table_param.get_group_by_projector();
@ -287,6 +302,7 @@ int ObTableAccessParam::init(
// vectorize requires blockscan is enabled(_pushdown_storage_level > 0)
iter_param_.vectorized_enabled_ = nullptr != get_op() && get_op()->is_vectorized();
iter_param_.limit_prefetch_ = (nullptr == op_filters_ || op_filters_->empty());
iter_param_.is_mds_query_ = scan_param.is_mds_query_;
if (iter_param_.is_use_column_store() &&
nullptr != table_param.get_read_info().get_cg_idxs() &&
!iter_param_.need_fill_group_idx()) { // not use column store in group rescan

8
src/storage/access/ob_table_access_param.h
View File

@ -215,6 +215,7 @@ public:
bool has_lob_column_out_;
bool is_for_foreign_check_;
bool limit_prefetch_;
bool is_mds_query_;
bool is_non_unique_local_index_;
int64_t ss_rowkey_prefix_cnt_;
sql::ObStoragePushdownFlag pd_storage_flag_;
@ -229,8 +230,11 @@ public:
void reset();
OB_INLINE bool is_valid() const { return is_inited_ && iter_param_.is_valid(); }
// used for query
int init(const ObTableScanParam &scan_param, const ObTabletHandle &tablet_handle);
// used for merge
int init(
const ObTableScanParam &scan_param,
const ObTabletHandle *tablet_handle,
const ObITableReadInfo *rowkey_read_info = nullptr);
// used for merge
int init_merge_param(const uint64_t table_id,
const common::ObTabletID &tablet_id,
const ObITableReadInfo &read_info,

15
src/storage/access/ob_table_scan_iterator.cpp
View File

@ -134,10 +134,13 @@ void ObTableScanIterator::reuse_row_iters()
int ObTableScanIterator::prepare_table_param(const ObTabletHandle &tablet_handle)
{
int ret = OB_SUCCESS;
if (nullptr == scan_param_ || nullptr == scan_param_->table_param_) {
const ObTablet *tablet = tablet_handle.get_obj();
if (OB_ISNULL(scan_param_)
|| OB_ISNULL(scan_param_->table_param_)
|| OB_ISNULL(tablet)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument", K(ret), KP(scan_param_));
} else if (OB_FAIL(main_table_param_.init(*scan_param_, tablet_handle))) {
STORAGE_LOG(WARN, "invalid args", K(ret), KP(scan_param_), K(tablet_handle));
} else if (OB_FAIL(main_table_param_.init(*scan_param_, &tablet_handle))) {
STORAGE_LOG(WARN, "failed to init main table param", K(ret));
} else if (nullptr != cached_iter_node_) {
main_table_param_.set_use_global_iter_pool();
@ -207,9 +210,9 @@ void ObTableScanIterator::try_release_cached_iter_node(const ObQRIterType rescan
int ObTableScanIterator::prepare_table_context()
{
int ret = OB_SUCCESS;
if (nullptr == scan_param_ || nullptr == scan_param_->table_param_) {
if (OB_ISNULL(scan_param_) || OB_ISNULL(scan_param_->table_param_)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument", K(ret), KP(scan_param_));
STORAGE_LOG(WARN, "invalid args", K(ret), KP(scan_param_));
} else {
ObVersionRange trans_version_range;
trans_version_range.multi_version_start_ = 0;
@ -629,7 +632,6 @@ int ObTableScanIterator::get_next_row(blocksstable::ObDatumRow *&row)
} else if (OB_ISNULL(main_iter_)) {
ret = OB_ITER_END;
} else {
ObDatum *trans_info_datums = nullptr;
if (scan_param_->op_ != nullptr) {
scan_param_->op_->clear_datum_eval_flag();
scan_param_->op_->reset_trans_info_datum();
@ -662,7 +664,6 @@ int ObTableScanIterator::get_next_rows(int64_t &count, int64_t capacity)
} else if (OB_ISNULL(main_iter_)) {
ret = OB_ITER_END;
} else {
ObDatum *trans_info_datums = nullptr;
if (scan_param_->op_ != nullptr) {
scan_param_->op_->clear_datum_eval_flag();
scan_param_->op_->reset_trans_info_datum();

4
src/storage/access/ob_table_scan_iterator.h
View File

@ -50,8 +50,8 @@ class ObTableScanIterator : public common::ObNewRowIterator
public:
ObTableScanIterator();
virtual ~ObTableScanIterator();
int init( ObTableScanParam &scan_param, const ObTabletHandle &tablet_handle);
int switch_param( ObTableScanParam &scan_param, const ObTabletHandle &tablet_handle);
int init(ObTableScanParam &scan_param, const ObTabletHandle &tablet_handle);
int switch_param(ObTableScanParam &scan_param, const ObTabletHandle &tablet_handle);
int get_next_row(blocksstable::ObDatumRow *&row);
virtual int get_next_row(common::ObNewRow *&row) override;
virtual int get_next_row() override { blocksstable::ObDatumRow *r = nullptr; return get_next_row(r); }

8
src/storage/access/ob_table_scan_range.h
View File

@ -48,10 +48,10 @@ private:
int init_ranges(const common::ObIArray<common::ObNewRange> &ranges,
const common::ObQueryFlag &scan_flag,
const blocksstable::ObStorageDatumUtils *datum_utils);
int init_ranges_in_skip_scan(const common::ObIArray<common::ObNewRange> &ranges,
const common::ObIArray<common::ObNewRange> &skip_scan_ranges,
const common::ObQueryFlag &scan_flag,
const blocksstable::ObStorageDatumUtils *datum_utils);
int init_ranges_in_skip_scan(const common::ObIArray<common::ObNewRange> &ranges,
const common::ObIArray<common::ObNewRange> &skip_scan_ranges,
const common::ObQueryFlag &scan_flag,
const blocksstable::ObStorageDatumUtils *datum_utils);
int always_false(const common::ObNewRange &range, bool &is_false);
private:
struct ObSkipScanWrappedRange

16
src/storage/backup/ob_backup_reader.cpp
View File

@ -23,6 +23,7 @@
#include "storage/blocksstable/ob_logic_macro_id.h"
#include "lib/utility/ob_tracepoint.h"
#include "observer/ob_server_event_history_table_operator.h"
#include "storage/tablet/ob_mds_schema_helper.h"
using namespace oceanbase::blocksstable;
using namespace oceanbase::storage;
@ -124,17 +125,8 @@ int ObTabletLogicMacroIdReader::init(const common::ObTabletID &tablet_id, const
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid argument", K(ret), K(tablet_id), K(table_key), K(batch_size));
} else if (FALSE_IT(datum_range_.set_whole_range())) {
} else {
if (sstable.is_normal_cg_sstable()) {
if (OB_FAIL(MTL(ObTenantCGReadInfoMgr *)->get_index_read_info(index_read_info))) {
LOG_WARN("failed to get index read info from ObTenantCGReadInfoMgr", K(ret), K(sstable));
}
} else {
index_read_info = &tablet_handle.get_obj()->get_rowkey_read_info();
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(tablet_handle.get_obj()->get_sstable_read_info(&sstable, index_read_info))) {
LOG_WARN("failed to get index read info ", KR(ret), K(sstable));
} else if (OB_FAIL(meta_iter_.open(datum_range_,
ObMacroBlockMetaType::DATA_BLOCK_META,
sstable,
@ -633,7 +625,7 @@ int ObSSTableMetaBackupReader::get_meta_data(blocksstable::ObBufferReader &buffe
ObBackupSSTableMeta backup_sstable_meta;
backup_sstable_meta.tablet_id_ = tablet_id_;
if ((backup_data_type_.is_major_backup() && !sstable_ptr->is_major_sstable())
|| (backup_data_type_.is_minor_backup() && !sstable_ptr->is_minor_sstable() && !sstable_ptr->is_ddl_dump_sstable())) {
|| (backup_data_type_.is_minor_backup() && !sstable_ptr->is_minor_sstable() && !sstable_ptr->is_ddl_dump_sstable() && !sstable_ptr->is_mds_sstable())) {
ret = OB_ERR_SYS;
LOG_WARN("get incorrect table type", K(ret), K(i), K_(backup_data_type), KP(sstable_ptr));
} else if (OB_FAIL(tablet->build_migration_sstable_param(table_key, backup_sstable_meta.sstable_meta_))) {

29
src/storage/backup/ob_backup_utils.cpp
View File

@ -39,6 +39,7 @@
#include "storage/column_store/ob_column_oriented_sstable.h"
#include "storage/backup/ob_backup_data_store.h"
#include "storage/ddl/ob_ddl_merge_task.h"
#include "storage/tablet/ob_mds_schema_helper.h"
#include <algorithm>
@ -106,14 +107,19 @@ int ObBackupUtils::get_sstables_by_data_type(const storage::ObTabletHandle &tabl
} else if (backup_data_type.is_minor_backup()) {
const storage::ObSSTableArray *minor_sstable_array_ptr = NULL;
const storage::ObSSTableArray *ddl_sstable_array_ptr = NULL;
const storage::ObSSTableArray *mds_sstable_array_ptr = NULL;
minor_sstable_array_ptr = &tablet_table_store.get_minor_sstables();
ddl_sstable_array_ptr = &tablet_table_store.get_ddl_sstables();
mds_sstable_array_ptr = &tablet_table_store.get_mds_sstables();
ObArray<storage::ObSSTableWrapper> minor_sstable_array;
ObArray<storage::ObSSTableWrapper> ddl_sstable_array;
ObArray<storage::ObSSTableWrapper> mds_sstable_array;
if (OB_FAIL(minor_sstable_array_ptr->get_all_table_wrappers(minor_sstable_array))) {
LOG_WARN("failed to get all tables", K(ret), KPC(minor_sstable_array_ptr));
} else if (OB_FAIL(ddl_sstable_array_ptr->get_all_table_wrappers(ddl_sstable_array, true/*unpack_table*/))) {
LOG_WARN("failed to get all tables", K(ret), KPC(ddl_sstable_array_ptr));
} else if (OB_FAIL(mds_sstable_array_ptr->get_all_table_wrappers(mds_sstable_array, true/*unpack_table*/))) {
LOG_WARN("failed to get all tables", K(ret), KPC(mds_sstable_array_ptr));
} else if (OB_FAIL(check_tablet_minor_sstable_validity_(tablet_handle, minor_sstable_array))) {
LOG_WARN("failed to check tablet minor sstable validity", K(ret), K(tablet_handle), K(minor_sstable_array));
} else if (OB_FAIL(check_tablet_ddl_sstable_validity_(tablet_handle, ddl_sstable_array))) {
@ -122,6 +128,8 @@ int ObBackupUtils::get_sstables_by_data_type(const storage::ObTabletHandle &tabl
LOG_WARN("failed to append", K(ret), K(minor_sstable_array));
} else if (OB_FAIL(append(sstable_array, ddl_sstable_array))) {
LOG_WARN("failed to append", K(ret), K(ddl_sstable_array));
} else if (OB_FAIL(append(sstable_array, mds_sstable_array))) {
LOG_WARN("failed to append", K(ret), K(mds_sstable_array));
}
} else if (backup_data_type.is_major_backup()) {
const storage::ObSSTableArray *major_sstable_array_ptr = NULL;
@ -129,7 +137,9 @@ int ObBackupUtils::get_sstables_by_data_type(const storage::ObTabletHandle &tabl
ObITable *last_major_sstable_ptr = NULL;
ObArenaAllocator tmp_allocator("backup_medium", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID());
bool with_major_sstable = true;
compaction::ObMediumCompactionInfoList medium_info_list;
const share::ObLSID &ls_id = tablet_handle.get_obj()->get_ls_id();
const common::ObTabletID &tablet_id = tablet_handle.get_obj()->get_tablet_id();
const int64_t last_compaction_scn = tablet_handle.get_obj()->get_last_compaction_scn();
ObSSTableWrapper major_wrapper;
if (OB_ISNULL(last_major_sstable_ptr = major_sstable_array_ptr->get_boundary_table(true /*last*/))) {
@ -140,13 +150,11 @@ int ObBackupUtils::get_sstables_by_data_type(const storage::ObTabletHandle &tabl
ret = OB_ERR_UNEXPECTED;
LOG_WARN("last major sstable should not be null", K(ret), K(tablet_handle));
}
} else if (OB_FAIL(tablet_handle.get_obj()->get_medium_info_list(tmp_allocator, medium_info_list))) {
LOG_WARN("failed to load medium info list", K(ret));
} else if (medium_info_list.get_last_compaction_scn() > 0
&& medium_info_list.get_last_compaction_scn() != last_major_sstable_ptr->get_snapshot_version()) {
} else if (OB_UNLIKELY(last_compaction_scn > 0
&& last_compaction_scn != last_major_sstable_ptr->get_snapshot_version())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("medium list is invalid for last major sstable", K(ret), K(medium_info_list),
KPC(last_major_sstable_ptr), K(tablet_handle));
LOG_WARN("extra medium info is invalid for last major sstable", K(ret), K(ls_id), K(tablet_id),
K(last_compaction_scn), KPC(last_major_sstable_ptr), K(tablet_handle));
} else if (last_major_sstable_ptr->is_co_sstable()) {
if (OB_FAIL(static_cast<ObCOSSTableV2 *>(last_major_sstable_ptr)->get_all_tables(sstable_array))) {
LOG_WARN("failed to get all cg tables from co table", K(ret), KPC(last_major_sstable_ptr));
@ -182,7 +190,7 @@ int ObBackupUtils::fetch_macro_block_logic_id_list(const storage::ObTabletHandle
logic_id_list.reset();
ObArenaAllocator allocator;
ObDatumRange datum_range;
const storage::ObITableReadInfo *index_read_info = &tablet_handle.get_obj()->get_rowkey_read_info();
const storage::ObITableReadInfo *index_read_info = NULL;
SMART_VAR(ObSSTableSecMetaIterator, meta_iter)
{
@ -190,9 +198,8 @@ int ObBackupUtils::fetch_macro_block_logic_id_list(const storage::ObTabletHandle
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid args", K(ret), K(sstable));
} else if (FALSE_IT(datum_range.set_whole_range())) {
} else if (sstable.is_normal_cg_sstable() &&
OB_FAIL(MTL(ObTenantCGReadInfoMgr *)->get_index_read_info(index_read_info))) {
LOG_WARN("failed to get index read info from ObTenantCGReadInfoMgr", K(ret), K(sstable));
} else if (OB_FAIL(tablet_handle.get_obj()->get_sstable_read_info(&sstable, index_read_info))) {
LOG_WARN("failed to get index read info ", KR(ret), K(sstable));
} else if (OB_FAIL(meta_iter.open(datum_range,
ObMacroBlockMetaType::DATA_BLOCK_META,
sstable,

2
src/storage/blocksstable/index_block/ob_index_block_tree_cursor.cpp
View File

@ -356,7 +356,7 @@ int ObIndexBlockTreeCursor::init(
} else if (OB_UNLIKELY(!sstable.is_normal_cg_sstable() && sstable_rowkey_col_cnt != read_info->get_rowkey_count())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("Rowkey column count not match between read info and sstable",
K(ret), K(read_info), K(sstable_rowkey_col_cnt));
K(ret), KPC(read_info), K(sstable_rowkey_col_cnt));
} else {
tenant_id_ = MTL_ID();
ObRowStoreType root_row_store_type

6
src/storage/blocksstable/ob_block_manager.cpp
View File

@ -990,10 +990,6 @@ void ObBlockManager::mark_and_sweep()
LOG_WARN("block manager not init", K(ret));
} else if (!is_mark_sweep_enabled()) {
LOG_INFO("mark and sweep is disabled, do not mark and sweep this round");
} else if (!ObServerCheckpointSlogHandler::get_instance().is_started()) {
if (REACH_TIME_INTERVAL(10 * 1000 * 1000 /* 10s */)) {
LOG_WARN("slog replay hasn't finished, this task can't start", K(ret));
}
} else {
if (OB_FAIL(mark_info.init(ObModIds::OB_STORAGE_FILE_BLOCK_REF, OB_SERVER_TENANT_ID))) {
LOG_WARN("fail to init mark info, ", K(ret));
@ -1164,7 +1160,7 @@ int ObBlockManager::mark_tenant_blocks(
} else {
LOG_WARN("fail to get next in-memory tablet", K(ret));
}
} else if (handle.get_obj()->is_old_tablet()) {
} else if (handle.get_obj()->get_version() < ObTabletBlockHeader::TABLET_VERSION_V3) {
if (OB_FAIL(mark_tablet_meta_blocks(mark_info, handle, macro_id_set, tmp_status))) {
LOG_WARN("fail to mark tablet meta blocks", K(ret));
} else if (OB_FAIL(mark_sstable_blocks(mark_info, handle, macro_id_set, tmp_status))) {

9
src/storage/blocksstable/ob_datum_row.cpp
View File

@ -251,6 +251,8 @@ int ObDatumRow::init(ObIAllocator &allocator, const int64_t capacity, char *tran
// trans_info_ptr maybe is nullptr,
// ObDatumRow does not care about the free of trans_info_ptr's memory
trans_info_ = trans_info_ptr;
STORAGE_LOG(DEBUG, "succeed to init datum row", K(ret), K_(count));
}
return ret;
@ -504,9 +506,12 @@ DEF_TO_STRING(ObDatumRow)
J_COLON();
J_ARRAY_START();
for (int64_t i = 0; i < count_; ++i) {
databuff_printf(buf, buf_len, pos, "col_id=%ld:", i);
if (0 == i) {
databuff_printf(buf, buf_len, pos, "col_id=%ld:", i);
} else {
databuff_printf(buf, buf_len, pos, ", col_id=%ld:", i);
}
pos += storage_datums_[i].storage_to_string(buf + pos, buf_len - pos);
databuff_printf(buf, buf_len, pos, ",");
}
J_ARRAY_END();
}

5
src/storage/blocksstable/ob_datum_rowkey.cpp
View File

@ -200,15 +200,18 @@ DEF_TO_STRING(ObDatumRowkey)
if (nullptr != buf && buf_len >= 0) {
if (nullptr != datums_) {
for (int64_t i = 0; i < datum_cnt_; ++i) {
if (i > 0) {
databuff_printf(buf, buf_len, pos, ", ");
}
databuff_printf(buf, buf_len, pos, "idx=%ld:", i);
pos += datums_[i].storage_to_string(buf + pos, buf_len - pos);
databuff_printf(buf, buf_len, pos, ",");
}
} else {
J_EMPTY_OBJ();
}
}
J_ARRAY_END();
J_COMMA();
J_KV(K_(store_rowkey));
J_OBJ_END();
return pos;

12
src/storage/blocksstable/ob_logic_macro_id.cpp
View File

@ -77,7 +77,7 @@ bool ObLogicMacroBlockId::operator==(const ObLogicMacroBlockId &other) const
return data_seq_ == other.data_seq_
&& logic_version_ == other.logic_version_
&& tablet_id_ == other.tablet_id_
&& column_group_idx_ == other.column_group_idx_;
&& info_ == other.info_;
}
bool ObLogicMacroBlockId::operator!=(const ObLogicMacroBlockId &other) const
@ -104,6 +104,10 @@ bool ObLogicMacroBlockId::operator<(const ObLogicMacroBlockId &other) const
bool_ret = true;
} else if (column_group_idx_ > other.column_group_idx_) {
bool_ret = false;
} else if (!is_mds_ && other.is_mds_) {
bool_ret = true;
} else if (is_mds_ && !other.is_mds_) {
bool_ret = false;
}
return bool_ret;
}
@ -127,6 +131,10 @@ bool ObLogicMacroBlockId::operator>(const ObLogicMacroBlockId &other) const
bool_ret = false;
} else if (column_group_idx_ > other.column_group_idx_) {
bool_ret = true;
} else if (!is_mds_ && other.is_mds_) {
bool_ret = false;
} else if (is_mds_ && !other.is_mds_) {
bool_ret = true;
}
return bool_ret;
}
@ -135,7 +143,7 @@ void ObLogicMacroBlockId::reset() {
logic_version_ = 0;
data_seq_.reset();
tablet_id_ = 0;
column_group_idx_ = 0;
info_ = 0;
}
OB_SERIALIZE_MEMBER(ObLogicMacroBlockId,

5
src/storage/blocksstable/ob_logic_macro_id.h
View File

@ -144,7 +144,7 @@ public:
return data_seq_.is_valid() && logic_version_ > 0 && tablet_id_ > 0;
}
TO_STRING_KV(K_(data_seq), K_(logic_version), K_(tablet_id), K_(column_group_idx));
TO_STRING_KV(K_(data_seq), K_(logic_version), K_(tablet_id), K_(column_group_idx), K_(is_mds));
public:
ObMacroDataSeq data_seq_;
@ -155,7 +155,8 @@ public:
uint64_t info_;
struct {
uint64_t column_group_idx_ : 16;
uint64_t reserved_ : 48;
bool is_mds_ : 1;
uint64_t reserved_ : 47;
};
};
OB_UNIS_VERSION(LOGIC_BLOCK_ID_VERSION);

1
src/storage/blocksstable/ob_macro_block.cpp
View File

@ -489,6 +489,7 @@ int ObMacroBlock::get_macro_block_meta(ObDataMacroBlockMeta &macro_meta)
macro_meta.val_.logic_id_.column_group_idx_= spec_->get_table_cg_idx();
macro_meta.val_.logic_id_.data_seq_.macro_data_seq_ = macro_header_.fixed_header_.data_seq_;
macro_meta.val_.logic_id_.tablet_id_ = spec_->get_tablet_id().id();
macro_meta.val_.logic_id_.is_mds_ = is_mds_merge(spec_->get_merge_type());
macro_meta.val_.macro_id_ = ObIndexBlockRowHeader::DEFAULT_IDX_ROW_MACRO_ID;
macro_meta.val_.rowkey_count_ = macro_header_.fixed_header_.rowkey_column_count_;
macro_meta.val_.compressor_type_ = spec_->get_compressor_type();

6
src/storage/blocksstable/ob_macro_block_writer.cpp
View File

@ -904,6 +904,11 @@ int ObMacroBlockWriter::check_order(const ObDatumRow &row)
if (cur_row_version < last_row_version) {
ret = OB_ROWKEY_ORDER_ERROR;
STORAGE_LOG(ERROR, "cur row version is less than last row version, ", K(ret), K(cur_row_version), K(last_row_version));
// TODO: @luhaopeng.lhp must delete before merge in master.
dump_micro_block(*micro_writer_); // print micro block have output
sleep(2);
ob_abort();
} else if (cur_row_version == last_row_version) {
int64_t last_row_sql_seq = last_key_.datums_[sql_sequence_col_idx].get_int();
if (cur_sql_sequence == last_row_sql_seq) {
@ -1389,6 +1394,7 @@ int ObMacroBlockWriter::flush_macro_block(ObMacroBlock &macro_block)
cur_logic_id.column_group_idx_ = data_store_desc_->get_table_cg_idx();
cur_logic_id.data_seq_.macro_data_seq_ = current_macro_seq_;
cur_logic_id.tablet_id_ = data_store_desc_->get_tablet_id().id();
cur_logic_id.is_mds_ = is_mds_merge(data_store_desc_->get_merge_type());
ObMacroBlockHandle &macro_handle = macro_handles_[current_index_];
ObMacroBlockHandle &prev_handle = macro_handles_[(current_index_ + 1) % 2];

64
src/storage/blocksstable/ob_shared_macro_block_manager.cpp
View File

@ -26,6 +26,7 @@
#include "storage/blocksstable/index_block/ob_index_block_builder.h"
#include "storage/blocksstable/index_block/ob_sstable_sec_meta_iterator.h"
#include "storage/tablet/ob_tablet_create_delete_helper.h"
#include "storage/tablet/ob_mds_schema_helper.h"
#include "storage/ls/ob_ls.h"
#include "share/ob_ls_id.h"
#include "storage/tx_storage/ob_ls_service.h"
@ -766,27 +767,48 @@ int ObSharedMacroBlockMgr::prepare_data_desc(
ObWholeDataStoreDesc &data_desc) const
{
int ret = OB_SUCCESS;
ObArenaAllocator tmp_arena("ShrBlkMgrTmp");
ObStorageSchema *storage_schema = nullptr;
data_desc.reset();
if (OB_FAIL(tablet.load_storage_schema(tmp_arena, storage_schema))) {
LOG_WARN("fail to load storage schema", K(ret), K(tablet));
} else if (OB_FAIL(data_desc.init(
*storage_schema,
tablet.get_tablet_meta().ls_id_,
tablet.get_tablet_meta().tablet_id_,
merge_type,
snapshot_version,
cluster_version,
end_scn))) {
LOG_WARN("failed to init static desc", K(ret), KPC(storage_schema),
K(tablet), "merge_type", merge_type_to_str(merge_type), K(snapshot_version), K(cluster_version));
} else if (OB_FAIL(data_desc.get_desc().update_basic_info_from_macro_meta(basic_meta))) {
// overwrite the encryption related memberships, otherwise these memberships of new sstable may differ
// from that of old sstable, since the encryption method of one tablet may change before defragmentation
LOG_WARN("failed to update basic info from macro_meta", KR(ret), K(basic_meta));
if (is_mds_merge(merge_type)) {
const ObStorageSchema *storage_schema = ObMdsSchemaHelper::get_instance().get_storage_schema();
if (OB_ISNULL(storage_schema)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("storage schema is null", K(ret), KP(storage_schema));
} else if (OB_UNLIKELY(!storage_schema->is_valid())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("mds storage schema is invalid", K(ret), KP(storage_schema), KPC(storage_schema));
} else if (OB_FAIL(data_desc.init(
*storage_schema,
tablet.get_tablet_meta().ls_id_,
tablet.get_tablet_meta().tablet_id_,
merge_type,
snapshot_version,
cluster_version,
end_scn))) {
LOG_WARN("failed to init static desc", K(ret), KPC(storage_schema),
K(tablet), "merge_type", merge_type_to_str(merge_type), K(snapshot_version), K(cluster_version));
}
} else {
ObArenaAllocator tmp_arena("ShrBlkMgrTmp");
ObStorageSchema *storage_schema = nullptr;
if (OB_FAIL(tablet.load_storage_schema(tmp_arena, storage_schema))) {
LOG_WARN("fail to load storage schema", K(ret), K(tablet));
} else if (OB_FAIL(data_desc.init(
*storage_schema,
tablet.get_tablet_meta().ls_id_,
tablet.get_tablet_meta().tablet_id_,
merge_type,
snapshot_version,
cluster_version,
end_scn))) {
LOG_WARN("failed to init static desc", K(ret), KPC(storage_schema),
K(tablet), "merge_type", merge_type_to_str(merge_type), K(snapshot_version), K(cluster_version));
} else if (OB_FAIL(data_desc.get_desc().update_basic_info_from_macro_meta(basic_meta))) {
// overwrite the encryption related memberships, otherwise these memberships of new sstable may differ
// from that of old sstable, since the encryption method of one tablet may change before defragmentation
LOG_WARN("failed to update basic info from macro_meta", KR(ret), K(basic_meta));
}
ObTabletObjLoadHelper::free(tmp_arena, storage_schema);
}
ObTabletObjLoadHelper::free(tmp_arena, storage_schema);
return ret;
}
@ -801,6 +823,10 @@ int ObSharedMacroBlockMgr::parse_merge_type(const ObSSTable &sstable, ObMergeTyp
: ObMergeType::MAJOR_MERGE;
} else if (sstable.is_minor_sstable()) {
merge_type = ObMergeType::MINOR_MERGE;
} else if (sstable.is_mds_mini_sstable()) {
merge_type = ObMergeType::MDS_MINI_MERGE;
} else if (sstable.is_mds_minor_sstable()) {
merge_type = ObMergeType::MDS_MINOR_MERGE;
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sstable type is unexpected", K(ret), K(sstable));

101
src/storage/blocksstable/ob_sstable.cpp
View File

@ -306,6 +306,65 @@ int ObSSTable::init(const ObTabletCreateSSTableParam &param, common::ObArenaAllo
return ret;
}
int ObSSTable::copy_from_old_sstable(const ObSSTable &src, common::ObArenaAllocator &allocator, ObSSTable *&dst)
{
int ret = OB_SUCCESS;
ObSSTable *sstable = nullptr;
if (OB_UNLIKELY(!src.is_valid() || !src.is_loaded())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(src));
} else if (OB_FAIL(src.inner_deep_copy_and_inc_macro_ref(allocator, sstable))) {
LOG_WARN("fail to inner copy and inc macro ref", K(ret), K(src));
} else if (!sstable->is_co_sstable() || sstable->meta_->cg_sstables_.count() == 0) {
// nothing to do and skip it.
} else {
ObSEArray<ObITable *, 64> cg_sstables;
for (int64_t i = 0; OB_SUCC(ret) && i < sstable->meta_->cg_sstables_.count(); ++i) {
ObSSTable *table = sstable->meta_->cg_sstables_.at(i);
ObSSTable *loaded_table = nullptr;
ObSSTable *copied_table = nullptr;
ObStorageMetaHandle handle;
if (OB_ISNULL(table) || OB_UNLIKELY(!table->is_cg_sstable())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected error, cg table is nullptr", K(ret), K(i), KPC(table), KPC(sstable));
} else if (table->is_loaded()) {
loaded_table = table;
} else if (OB_FAIL(ObTabletTableStore::load_sstable(table->get_addr(), false/*is_co_sstable*/, handle))) {
LOG_WARN("fail to load cg sstable", K(ret), KPC(table));
} else if (OB_FAIL(handle.get_sstable(loaded_table))) {
LOG_WARN("fail to get sstable", K(ret), K(handle));
}
if (FAILEDx(loaded_table->inner_deep_copy_and_inc_macro_ref(allocator, copied_table))) {
LOG_WARN("fail to inner copy and inc macro ref", K(ret), KPC(loaded_table));
} else if (OB_FAIL(cg_sstables.push_back(copied_table))) {
LOG_WARN("fail to push back", K(ret), KPC(copied_table));
}
}
if (OB_SUCC(ret)) {
sstable->meta_->cg_sstables_.reset();
if (OB_FAIL(sstable->meta_->cg_sstables_.init_empty_array_for_cg(allocator, cg_sstables.count()))) {
LOG_WARN("fail to init cg sstables", K(ret), K(cg_sstables));
} else if (OB_FAIL(sstable->meta_->cg_sstables_.add_tables_for_cg_without_deep_copy(cg_sstables))) {
LOG_WARN("fail to add tables for cg without deep copy", K(ret), K(cg_sstables));
}
}
if (OB_FAIL(ret)) {
for (int64_t i = 0; i < cg_sstables.count(); ++i) {// ingore error code
cg_sstables.at(i)->~ObITable();
}
}
}
if (OB_FAIL(ret)) {
if (OB_NOT_NULL(sstable)) {
sstable->reset();
}
} else {
dst = sstable;
FLOG_INFO("succeeded to init sstable", K(ret), KPC(dst));
}
return ret;
}
void ObSSTable::reset()
{
LOG_DEBUG("reset sstable.", KP(this), K(key_), K(is_tmp_sstable_));
@ -2032,5 +2091,47 @@ int ObSSTable::init_sstable_meta(
}
int ObSSTable::inner_deep_copy_and_inc_macro_ref(
common::ObIAllocator &allocator,
ObSSTable *&sstable) const
{
int ret = OB_SUCCESS;
bool inc_success = false;
const int64_t deep_copy_size = get_deep_copy_size();
char *buf = nullptr;
ObIStorageMetaObj *meta_obj = nullptr;
ObSSTable *table = nullptr;
if (OB_ISNULL(buf = static_cast<char *>(allocator.alloc(deep_copy_size)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to allocate memory for deep copy sstable", K(ret), K(deep_copy_size));
} else if (OB_FAIL(deep_copy(buf, deep_copy_size, meta_obj))) {
LOG_WARN("fail to inner deep copy sstable", K(ret));
} else {
table = static_cast<ObSSTable *>(meta_obj);
table->addr_.set_mem_addr(0, deep_copy_size);
if (OB_FAIL(table->inc_macro_ref(inc_success))) {
LOG_WARN("fail to add macro ref", K(ret), K(inc_success));
} else if (!inc_success) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected error, increase macro ref failed", K(ret), K(inc_success));
} else {
table->is_tmp_sstable_ = true;
}
}
if (OB_FAIL(ret)) {
if (OB_NOT_NULL(table)) {
table->reset();
}
if (OB_NOT_NULL(buf)) {
allocator.free(buf);
}
} else {
sstable = table;
LOG_INFO("succeeded to copy sstable and increase macro reference count", K(ret), KPC(sstable));
}
return ret;
}
} // namespace blocksstable
} // namespace oceanbase

2
src/storage/blocksstable/ob_sstable.h
View File

@ -132,6 +132,7 @@ public:
virtual int64_t get_ref() const override;
virtual int init(const ObTabletCreateSSTableParam &param, common::ObArenaAllocator *allocator);
static int copy_from_old_sstable(const ObSSTable &old_sstable, common::ObArenaAllocator &allocator, ObSSTable *&sstable);
void reset();
// Query interfaces
@ -360,6 +361,7 @@ protected:
int deserialize_fixed_struct(const char *buf, const int64_t data_len, int64_t &pos);
int64_t get_sstable_fix_serialize_size() const;
int64_t get_sstable_fix_serialize_payload_size() const;
int inner_deep_copy_and_inc_macro_ref(common::ObIAllocator &allocator, ObSSTable *&sstable) const;
protected:
static const int64_t SSTABLE_VERSION = 1;
static const int64_t SSTABLE_VERSION_V2 = 2;

1
src/storage/blocksstable/ob_sstable_meta.cpp
View File

@ -588,6 +588,7 @@ void ObSSTableMeta::reset()
basic_meta_.reset();
column_checksums_ = nullptr;
column_checksum_count_ = 0;
cg_sstables_.reset();
tx_ctx_.reset();
is_inited_ = false;
}

107
src/storage/compaction/ob_basic_tablet_merge_ctx.cpp
View File

@ -24,6 +24,8 @@
#include "storage/ob_gc_upper_trans_helper.h"
#include "ob_medium_list_checker.h"
#include "share/schema/ob_tenant_schema_service.h"
#include "storage/tablet/ob_mds_schema_helper.h"
#include "storage/tablet/ob_mds_scan_param_helper.h"
namespace oceanbase
{
@ -62,7 +64,8 @@ ObStaticMergeParam::ObStaticMergeParam(ObTabletMergeDagParam &dag_param)
report_(nullptr),
snapshot_info_(),
tx_id_(0),
multi_version_column_descs_()
multi_version_column_descs_(),
is_backfill_(false)
{
merge_scn_.set_max();
}
@ -70,12 +73,6 @@ ObStaticMergeParam::ObStaticMergeParam(ObTabletMergeDagParam &dag_param)
void ObStaticMergeParam::reset()
{
tables_handle_.reset();
if (nullptr != schema_) {
schema_->~ObStorageSchema();
schema_ = nullptr;
// TODO(@lixia.yq): ensure that the buffer corresponding to storage schema is always allocated by ObArenaAllocator
// otherwise there will be memory leak here.
}
report_ = nullptr;
rowkey_read_info_ = nullptr;
co_major_merge_type_ = ObCOMajorMergePolicy::INVALID_CO_MAJOR_MERGE_TYPE;
@ -117,7 +114,11 @@ int ObStaticMergeParam::init_static_info(
ObTabletHandle &tablet_handle)
{
int ret = OB_SUCCESS;
rowkey_read_info_ = static_cast<const ObRowkeyReadInfo *>(&(tablet_handle.get_obj()->get_rowkey_read_info()));
if (is_mds_minor_merge(get_merge_type())) {
rowkey_read_info_ = ObMdsSchemaHelper::get_instance().get_rowkey_read_info();
} else {
rowkey_read_info_ = static_cast<const ObRowkeyReadInfo *>(&(tablet_handle.get_obj()->get_rowkey_read_info()));
}
concurrent_cnt_ = concurrent_cnt;
if (OB_FAIL(init_multi_version_column_descs())) {
LOG_WARN("failed to init multi_version_column_descs", KR(ret));
@ -197,8 +198,10 @@ int ObStaticMergeParam::get_basic_info_from_result(
} else {
version_range_ = get_merge_table_result.version_range_;
scn_range_ = get_merge_table_result.scn_range_;
merge_scn_ = scn_range_.end_scn_;
snapshot_info_ = get_merge_table_result.snapshot_info_;
is_backfill_ = get_merge_table_result.is_backfill_;
merge_scn_ = get_merge_table_result.get_merge_scn();
if (is_major_or_meta_merge_type(get_merge_type())) {
// for major or meta, need set create_snapshot as last major/meta sstable
create_snapshot_version_ = tables_handle_.get_table(0)->get_snapshot_version();
@ -466,11 +469,22 @@ int ObBasicTabletMergeCtx::build_ctx_after_init()
void ObBasicTabletMergeCtx::destroy()
{
free_schema();
static_param_.reset(); // clear tables_handle before tablet_handle reset
info_collector_.destroy(mem_ctx_);
read_info_.reset();
}
void ObBasicTabletMergeCtx::free_schema()
{
if (nullptr != static_param_.schema_) {
static_param_.schema_->~ObStorageSchema();
static_param_.schema_ = nullptr;
// TODO(@lixia.yq): ensure that the buffer corresponding to storage schema is always allocated by ObArenaAllocator
// otherwise there will be memory leak here.
}
}
ObBasicTabletMergeCtx::ObBasicTabletMergeCtx(
ObTabletMergeDagParam &param,
common::ObArenaAllocator &allocator)
@ -826,6 +840,10 @@ ObITable::TableType ObBasicTabletMergeCtx::get_merged_table_type(
table_type = ObITable::TableType::MINI_SSTABLE;
} else if (DDL_KV_MERGE == get_merge_type()) {
table_type = ObITable::TableType::DDL_DUMP_SSTABLE;
} else if (MDS_MINI_MERGE == get_merge_type()) {
table_type = ObITable::TableType::MDS_MINI_SSTABLE;
} else if (MDS_MINOR_MERGE == get_merge_type()) {
table_type = ObITable::TableType::MDS_MINOR_SSTABLE;
} else { // MINOR_MERGE || HISTORY_MINOR_MERGE
table_type = ObITable::TableType::MINOR_SSTABLE;
}
@ -1064,51 +1082,74 @@ int ObBasicTabletMergeCtx::get_medium_compaction_info()
{
int ret = OB_SUCCESS;
ObTablet *tablet = get_tablet();
const share::ObLSID &ls_id = tablet->get_ls_id();
const common::ObTabletID &tablet_id = tablet->get_tablet_id();
ObArenaAllocator temp_allocator("GetMediumInfo", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); // for load medium info
ObMediumCompactionInfo medium_info;
ObMediumCompactionInfoKey medium_info_key(get_merge_version());
storage::ObTabletMediumInfoReader medium_info_reader(*tablet);
if (OB_UNLIKELY(get_tablet()->get_multi_version_start() > get_merge_version())) {
ObMediumCompactionInfo *medium_info = nullptr;
if (OB_UNLIKELY(tablet->get_multi_version_start() > get_merge_version())) {
ret = OB_SNAPSHOT_DISCARDED;
LOG_ERROR("multi version data is discarded, should not execute compaction now", K(ret),
"param", get_dag_param(), KPC(this));
} else if (OB_FAIL(medium_info_reader.init(temp_allocator))) {
LOG_WARN("failed to init medium info reader", K(ret), KPC(this));
} else if (OB_FAIL(medium_info_reader.get_specified_medium_info(temp_allocator, medium_info_key, medium_info))) {
LOG_WARN("failed to get specified scn info", K(ret), K(medium_info_key));
} else if (OB_UNLIKELY(!medium_info.is_valid())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("medium info is invalid", KR(ret), K(medium_info));
} else if (medium_info.contain_parallel_range_
} else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(temp_allocator, medium_info))) {
LOG_WARN("fail to alloc and new", K(ret));
} else {
SMART_VAR(ObTableScanParam, scan_param) {
storage::ObTabletMediumInfoReader medium_info_reader;
if (OB_FAIL(ObMdsScanParamHelper::build_medium_info_scan_param(
temp_allocator,
ls_id,
tablet_id,
scan_param))) {
LOG_WARN("fail to build scan param", K(ret), K(ls_id), K(tablet_id));
} else if (OB_FAIL(medium_info_reader.init(*tablet, scan_param))) {
LOG_WARN("failed to init medium info reader", K(ret), KPC(this));
} else if (OB_FAIL(medium_info_reader.get_specified_medium_info(temp_allocator, medium_info_key, *medium_info))) {
LOG_WARN("failed to get specified scn info", K(ret), K(medium_info_key));
} else if (OB_UNLIKELY(!medium_info->is_valid())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("medium info is invalid", KR(ret), K(medium_info));
}
}
}
if (OB_FAIL(ret)) {
} else if (medium_info->contain_parallel_range_
&& !parallel_merge_ctx_.is_valid()
&& OB_FAIL(parallel_merge_ctx_.init(medium_info))) {
LOG_WARN("failed to init parallel merge ctx", K(ret), K(medium_info));
&& OB_FAIL(parallel_merge_ctx_.init(*medium_info))) {
LOG_WARN("failed to init parallel merge ctx", K(ret), KPC(medium_info));
} else if (OB_FAIL(ObMediumListChecker::check_next_schedule_medium(
&medium_info, get_tables_handle().get_table(0)->get_snapshot_version(), true/*force_check*/))) {
medium_info, get_tables_handle().get_table(0)->get_snapshot_version(), true/*force_check*/))) {
LOG_WARN("failed to check medium info and last major sstable", KR(ret), K(medium_info), KPC(this));
} else if (OB_ISNULL(static_param_.schema_)) {
ObStorageSchema *storage_schema = nullptr;
if (OB_FAIL(ObStorageSchemaUtil::alloc_storage_schema(mem_ctx_.get_allocator(), storage_schema))) {
LOG_WARN("failed to alloc storage schema", K(ret));
} else if (OB_FAIL(storage_schema->init(mem_ctx_.get_allocator(), medium_info.storage_schema_))) {
} else if (OB_FAIL(storage_schema->init(mem_ctx_.get_allocator(), medium_info->storage_schema_))) {
LOG_WARN("failed to init storage schema from current medium info", K(ret), K(medium_info));
ObStorageSchemaUtil::free_storage_schema(mem_ctx_.get_allocator(), storage_schema);
} else {
static_param_.schema_ = storage_schema;
}
}
if (OB_SUCC(ret)) {
static_param_.schema_version_ = medium_info.storage_schema_.schema_version_;
static_param_.data_version_ = medium_info.data_version_;
static_param_.is_rebuild_column_store_ = (medium_info.medium_merge_reason_ == ObAdaptiveMergePolicy::REBUILD_COLUMN_GROUP);
static_param_.is_tenant_major_merge_ = medium_info.is_major_compaction();
if (medium_info.medium_compat_version_ >= ObMediumCompactionInfo::MEDIUM_COMPAT_VERSION_V4) {
static_param_.is_schema_changed_ = medium_info.is_schema_changed_;
static_param_.schema_version_ = medium_info->storage_schema_.schema_version_;
static_param_.data_version_ = medium_info->data_version_;
static_param_.is_rebuild_column_store_ = (medium_info->medium_merge_reason_ == ObAdaptiveMergePolicy::REBUILD_COLUMN_GROUP);
static_param_.is_tenant_major_merge_ = medium_info->is_major_compaction();
if (medium_info->medium_compat_version_ >= ObMediumCompactionInfo::MEDIUM_COMPAT_VERSION_V4) {
static_param_.is_schema_changed_ = medium_info->is_schema_changed_;
}
static_param_.merge_reason_ = (ObAdaptiveMergePolicy::AdaptiveMergeReason)medium_info.medium_merge_reason_;
static_param_.co_major_merge_type_ = static_cast<ObCOMajorMergePolicy::ObCOMajorMergeType>(medium_info.co_major_merge_type_);
FLOG_INFO("get storage schema to merge", "param", get_dag_param(), KPC_(static_param_.schema), K(medium_info));
static_param_.merge_reason_ = (ObAdaptiveMergePolicy::AdaptiveMergeReason)medium_info->medium_merge_reason_;
static_param_.co_major_merge_type_ = static_cast<ObCOMajorMergePolicy::ObCOMajorMergeType>(medium_info->co_major_merge_type_);
FLOG_INFO("get storage schema to merge", "param", get_dag_param(), KPC_(static_param_.schema), KPC(medium_info));
}
// always free medium info
ObTabletObjLoadHelper::free(temp_allocator, medium_info);
return ret;
}

6
src/storage/compaction/ob_basic_tablet_merge_ctx.h
View File

@ -60,7 +60,7 @@ public:
K_(sstable_logic_seq), K_(tables_handle), K_(is_rebuild_column_store), K_(is_schema_changed), K_(is_tenant_major_merge),
K_(read_base_version), K_(merge_scn), K_(need_parallel_minor_merge),
K_(progressive_merge_round), K_(progressive_merge_step), K_(progressive_merge_num),
K_(schema_version), KP_(schema), K_(multi_version_column_descs), K_(ls_handle), K_(snapshot_info), KP_(report));
K_(schema_version), KP_(schema), K_(multi_version_column_descs), K_(ls_handle), K_(snapshot_info), KP_(report), K_(is_backfill));
ObTabletMergeDagParam &dag_param_;
bool is_full_merge_; // full merge or increment merge
@ -94,6 +94,7 @@ public:
ObStorageSnapshotInfo snapshot_info_;
int64_t tx_id_;
common::ObSEArray<share::schema::ObColDesc, 2 * OB_ROW_DEFAULT_COLUMNS_COUNT> multi_version_column_descs_;
bool is_backfill_;
DISALLOW_COPY_AND_ASSIGN(ObStaticMergeParam);
};
@ -141,7 +142,7 @@ public:
const ObITableReadInfo *index_read_info,
const storage::ObStorageColumnGroupSchema *cg_schema = nullptr,
const uint16_t table_cg_idx = 0);
int get_ls_and_tablet();
virtual int get_ls_and_tablet();
void init_time_guard(const int64_t time) {
info_collector_.time_guard_.set_last_click_ts(time);
}
@ -222,6 +223,7 @@ protected:
return OB_SUCCESS;
}
virtual int prepare_schema();
virtual void free_schema();
virtual int cal_merge_param() { return static_param_.cal_minor_merge_param(false/*has_compaction_filter*/); }
int init_parallel_merge_ctx();
int init_static_param_and_desc();

1
src/storage/compaction/ob_compaction_util.cpp
View File

@ -27,6 +27,7 @@ const static char * ObMergeTypeStr[] = {
"DDL_KV_MERGE",
"BACKFILL_TX_MERGE",
"MDS_MINI_MERGE",
"MDS_MINOR_MERGE",
"EMPTY_MERGE_TYPE"
};

14
src/storage/compaction/ob_compaction_util.h
View File

@ -29,6 +29,7 @@ enum ObMergeType
DDL_KV_MERGE = 6, // only use for ddl dag
BACKFILL_TX_MERGE = 7,
MDS_MINI_MERGE = 8,
MDS_MINOR_MERGE = 9,
// add new merge type here
// fix merge_type_to_str & ObPartitionMergePolicy::get_merge_tables
MERGE_TYPE_MAX
@ -64,7 +65,8 @@ inline bool is_multi_version_merge(const ObMergeType &merge_type)
return MINOR_MERGE == merge_type
|| MINI_MERGE == merge_type
|| HISTORY_MINOR_MERGE == merge_type
|| BACKFILL_TX_MERGE == merge_type;
|| BACKFILL_TX_MERGE == merge_type
|| MDS_MINOR_MERGE == merge_type;
}
inline bool is_history_minor_merge(const ObMergeType &merge_type)
{
@ -86,10 +88,18 @@ inline bool is_backfill_tx_merge(const ObMergeType &merge_type)
{
return BACKFILL_TX_MERGE == merge_type;
}
inline bool is_mds_table_merge(const ObMergeType &merge_type)
inline bool is_mds_mini_merge(const ObMergeType &merge_type)
{
return MDS_MINI_MERGE == merge_type;
}
inline bool is_mds_minor_merge(const ObMergeType &merge_type)
{
return MDS_MINOR_MERGE == merge_type;
}
inline bool is_mds_merge(const ObMergeType &merge_type)
{
return is_mds_mini_merge(merge_type) || is_mds_minor_merge(merge_type);
}
enum ObMergeLevel : uint8_t
{

1
src/storage/compaction/ob_i_compaction_filter.cpp
View File

@ -128,6 +128,5 @@ int ObTransStatusFilter::filter(
return ret;
}
} // namespace compaction
} // namespace oceanbase

13
src/storage/compaction/ob_medium_compaction_info.cpp
View File

@ -315,16 +315,23 @@ int64_t ObParallelMergeInfo::to_string(char* buf, const int64_t buf_len) const
} else {
J_OBJ_START();
J_KV(K_(list_size), K_(compat));
J_COMMA();
if (list_size_ > 0) {
J_COMMA();
}
if (PARALLEL_INFO_VERSION_V0 == compat_) {
for (int i = 0; i < list_size_; ++i) {
if (i > 0) {
J_COMMA();
}
J_KV(K(i), "key", parallel_store_rowkey_list_[i]);
J_COMMA();
}
} else if (PARALLEL_INFO_VERSION_V1 == compat_) {
for (int i = 0; i < list_size_; ++i) {
if (i > 0) {
J_COMMA();
}
J_KV(K(i), "key", parallel_datum_rowkey_list_[i]);
J_COMMA();
}
}
J_OBJ_END();

71
src/storage/compaction/ob_medium_compaction_info.h
View File

@ -13,6 +13,7 @@
#ifndef OB_STORAGE_COMPACTION_MEDIUM_COMPACTION_INFO_H_
#define OB_STORAGE_COMPACTION_MEDIUM_COMPACTION_INFO_H_
#include "lib/ob_errno.h"
#include "storage/ob_storage_schema.h"
#include "lib/container/ob_array_array.h"
#include "observer/ob_server_struct.h"
@ -102,11 +103,11 @@ private:
ObIAllocator *allocator_;
};
struct ObMediumCompactionInfoKey final
{
public:
OB_UNIS_VERSION(1);
static constexpr uint8_t MAGIC_NUMBER = 0xFF; // if meet compat case, abort directly for now
public:
ObMediumCompactionInfoKey()
: medium_snapshot_(0)
@ -132,32 +133,52 @@ public:
return *this;
}
bool operator<(const ObMediumCompactionInfoKey& rhs) const
{
return medium_snapshot_ < rhs.medium_snapshot_;
}
bool operator<=(const ObMediumCompactionInfoKey& rhs) const
{
return medium_snapshot_ <= rhs.medium_snapshot_;
}
bool operator>(const ObMediumCompactionInfoKey& rhs) const
{
return medium_snapshot_ > rhs.medium_snapshot_;
}
bool operator>=(const ObMediumCompactionInfoKey& rhs) const
{
return medium_snapshot_ >= rhs.medium_snapshot_;
}
bool operator==(const ObMediumCompactionInfoKey& rhs) const
{
return medium_snapshot_ == rhs.medium_snapshot_;
}
bool operator!=(const ObMediumCompactionInfoKey& rhs) const
{
return medium_snapshot_ != rhs.medium_snapshot_;
}
int64_t get_medium_snapshot() const { return medium_snapshot_; }
int mds_serialize(char *buf, const int64_t buf_len, int64_t &pos) const {
int ret = OB_SUCCESS;
int64_t tmp = medium_snapshot_;
if (pos >= buf_len) {
ret = OB_BUF_NOT_ENOUGH;
} else {
buf[pos++] = MAGIC_NUMBER;
for (int64_t idx = 0; idx < 8 && OB_SUCC(ret); ++idx) {
if (pos >= buf_len) {
ret = OB_BUF_NOT_ENOUGH;
} else {
buf[pos++] = ((tmp >> (56 - 8 * idx)) & 0x00000000000000FF);
}
}
}
return ret;
}
int mds_deserialize(const char *buf, const int64_t buf_len, int64_t &pos) {
int ret = OB_SUCCESS;
int64_t tmp = 0;
uint8_t magic_number = 0;
if (pos >= buf_len) {
ret = OB_BUF_NOT_ENOUGH;
} else {
magic_number = buf[pos++];
if (magic_number != MAGIC_NUMBER) {
ob_abort();// compat case, just abort for fast fail
}
for (int64_t idx = 0; idx < 8 && OB_SUCC(ret); ++idx) {
if (pos >= buf_len) {
ret = OB_BUF_NOT_ENOUGH;
} else {
tmp <<= 8;
tmp |= (0x00000000000000FF & buf[pos++]);
}
}
}
if (OB_SUCC(ret)) {
medium_snapshot_ = tmp;
}
return ret;
}
int64_t mds_get_serialize_size() const { return sizeof(MAGIC_NUMBER) + sizeof(medium_snapshot_); }
TO_STRING_KV(K_(medium_snapshot));
private:
int64_t medium_snapshot_;

45
src/storage/compaction/ob_medium_compaction_mgr.cpp
View File

@ -473,7 +473,7 @@ int ObMediumCompactionInfoList::init(
int ObMediumCompactionInfoList::init(
common::ObIAllocator &allocator,
const ObExtraMediumInfo &extra_medium_info,
const ObTabletDumpedMediumInfo *medium_info_list)
const common::ObIArray<ObMediumCompactionInfo*> &medium_info_array)
{
int ret = OB_SUCCESS;
@ -482,32 +482,24 @@ int ObMediumCompactionInfoList::init(
LOG_WARN("init twice", K(ret));
} else {
allocator_ = &allocator;
if (nullptr == medium_info_list) {
// medium info is null, no need to copy
} else {
const common::ObIArray<ObMediumCompactionInfo*> &array = medium_info_list->medium_info_list_;
ObMediumCompactionInfo *medium_info = nullptr;
for (int64_t i = 0; OB_SUCC(ret) && i < array.count(); ++i) {
medium_info = nullptr;
const ObMediumCompactionInfo *src_medium_info = array.at(i);
if (OB_ISNULL(src_medium_info)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected error, medium info is null", K(ret), K(i), KP(src_medium_info));
} else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, medium_info))) {
LOG_WARN("failed to alloc and new", K(ret));
} else if (OB_FAIL(medium_info->init(allocator, *src_medium_info))) {
LOG_WARN("failed to copy medium info", K(ret), KPC(src_medium_info));
} else if (OB_UNLIKELY(!medium_info_list_.add_last(medium_info))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to add last", K(ret), KPC(medium_info));
}
ObMediumCompactionInfo *medium_info = nullptr;
for (int64_t i = 0; OB_SUCC(ret) && i < medium_info_array.count(); ++i) {
medium_info = nullptr;
const ObMediumCompactionInfo *src_medium_info = medium_info_array.at(i);
if (OB_ISNULL(src_medium_info)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected error, medium info is null", K(ret), K(i), KP(src_medium_info));
} else if (OB_FAIL(ObTabletObjLoadHelper::alloc_and_new(allocator, medium_info))) {
LOG_WARN("failed to alloc and new", K(ret));
} else if (OB_FAIL(medium_info->init(allocator, *src_medium_info))) {
LOG_WARN("failed to copy medium info", K(ret), KPC(src_medium_info));
} else if (OB_UNLIKELY(!medium_info_list_.add_last(medium_info))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to add last", K(ret), KPC(medium_info));
}
if (OB_FAIL(ret)) {
if (OB_NOT_NULL(medium_info)) {
medium_info->~ObMediumCompactionInfo();
allocator.free(medium_info);
}
}
if (OB_FAIL(ret)) {
ObTabletObjLoadHelper::free(allocator, medium_info);
}
}
@ -521,6 +513,7 @@ int ObMediumCompactionInfoList::init(
return ret;
}
void ObMediumCompactionInfoList::reset_list()
{
DLIST_REMOVE_ALL_NORET(info, medium_info_list_) {

Some files were not shown because too many files have changed in this diff Show More