diff --git a/deps/oblib/src/common/ob_common_types.h b/deps/oblib/src/common/ob_common_types.h index 609f69c7c8..87909f995c 100644 --- a/deps/oblib/src/common/ob_common_types.h +++ b/deps/oblib/src/common/ob_common_types.h @@ -57,7 +57,8 @@ struct ObQueryFlag #define OBSF_BIT_IS_SELECT_FOLLOWER 1 #define OBSF_BIT_ENABLE_LOB_PREFETCH 1 #define OBSF_BIT_IS_BARE_ROW_SCAN 1 -#define OBSF_BIT_RESERVED 24 +#define OBSF_BIT_MR_MV_SCAN 2 +#define OBSF_BIT_RESERVED 22 static const uint64_t OBSF_MASK_SCAN_ORDER = (0x1UL << OBSF_BIT_SCAN_ORDER) - 1; static const uint64_t OBSF_MASK_DAILY_MERGE = (0x1UL << OBSF_BIT_DAILY_MERGE) - 1; @@ -117,6 +118,12 @@ struct ObQueryFlag ReservedMode = 2, }; + enum MRMVScanMode + { + NormalMode = 0, + RefreshMode = 1, + RealTimeMode = 2, + }; union { uint64_t flag_; @@ -156,6 +163,7 @@ struct ObQueryFlag uint64_t is_mds_query_ : OBSF_BIT_IS_MDS_QUERY; uint64_t enable_lob_prefetch_ : OBSF_BIT_ENABLE_LOB_PREFETCH; uint64_t is_bare_row_scan_ : OBSF_BIT_IS_BARE_ROW_SCAN; // 1: to scan mult version row directly without compact. + uint64_t mr_mv_scan_ : OBSF_BIT_MR_MV_SCAN; // 0: normal table scan. 1. major refresh mview base table scan in refresh 2. major refresh rt-mview base table scan uint64_t reserved_ : OBSF_BIT_RESERVED; }; }; @@ -256,6 +264,9 @@ struct ObQueryFlag set_not_use_block_cache(); set_not_use_bloomfilter_cache(); } + inline bool is_mr_mview_refresh_base_scan() const { return RefreshMode == mr_mv_scan_; } + inline bool is_mr_rt_mview_base_scan() const { return RealTimeMode == mr_mv_scan_; } + inline bool is_mr_mview_query() const { return is_mr_mview_refresh_base_scan() || is_mr_rt_mview_base_scan(); } TO_STRING_KV("scan_order", scan_order_, "daily_merge", daily_merge_, @@ -288,6 +299,7 @@ struct ObQueryFlag "is_select_follower", is_select_follower_, "enable_lob_prefetch", enable_lob_prefetch_, "is_bare_row_scan", is_bare_row_scan_, + "mr_mv_scan", mr_mv_scan_, "reserved", reserved_); OB_UNIS_VERSION(1); }; diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h index 6d61d213f4..f8fa38d442 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h @@ -39,7 +39,7 @@ public: ObSessionDDLInfo() : is_ddl_(false), is_source_table_hidden_(false), is_dest_table_hidden_(false), is_heap_table_ddl_(false), is_ddl_check_default_value_bit_(false), is_mview_complete_refresh_(false), is_refreshing_mview_(false), - is_retryable_ddl_(false), is_dummy_ddl_for_inner_visibility_(false), reserved_bit_(0) + is_retryable_ddl_(false), is_dummy_ddl_for_inner_visibility_(false), is_major_refreshing_mview_(false), reserved_bit_(0) { } ~ObSessionDDLInfo() = default; @@ -61,6 +61,8 @@ public: bool is_retryable_ddl() const { return is_retryable_ddl_; } void set_is_dummy_ddl_for_inner_visibility(const bool flag) { is_dummy_ddl_for_inner_visibility_ = flag; } bool is_dummy_ddl_for_inner_visibility() const { return is_dummy_ddl_for_inner_visibility_; } + void set_major_refreshing_mview(const bool flag) { is_major_refreshing_mview_ = flag; } + bool is_major_refreshing_mview() const { return is_major_refreshing_mview_; } inline void reset() { ddl_info_ = 0; } TO_STRING_KV(K_(ddl_info)); OB_UNIS_VERSION(1); @@ -73,7 +75,8 @@ public: static const int64_t IS_REFRESHING_MVIEW_BIT = 1; static const int64_t IS_RETRYABLE_DDL_BIT = 1; static const int64_t IS_DUMMY_DDL_FOR_INNER_VISIBILITY_BIT = 1; - static const int64_t RESERVED_BIT = 64 - IS_DDL_BIT - 2 * IS_TABLE_HIDDEN_BIT - IS_HEAP_TABLE_DDL_BIT - IS_DDL_CHECK_DEFAULT_VALUE_BIT - IS_MVIEW_COMPLETE_REFRESH_BIT - IS_REFRESHING_MVIEW_BIT - IS_RETRYABLE_DDL_BIT - IS_DUMMY_DDL_FOR_INNER_VISIBILITY_BIT; + static const int64_t IS_MAJOR_REFRESHING_MVIEW_BIT = 1; + static const int64_t RESERVED_BIT = 64 - IS_DDL_BIT - 2 * IS_TABLE_HIDDEN_BIT - IS_HEAP_TABLE_DDL_BIT - IS_DDL_CHECK_DEFAULT_VALUE_BIT - IS_MVIEW_COMPLETE_REFRESH_BIT - IS_REFRESHING_MVIEW_BIT - IS_RETRYABLE_DDL_BIT - IS_DUMMY_DDL_FOR_INNER_VISIBILITY_BIT - IS_MAJOR_REFRESHING_MVIEW_BIT; union { uint64_t ddl_info_; struct { @@ -91,6 +94,7 @@ public: * When is_ddl_ is also enabled, it will override is_dummy_ddl_for_inner_visibility_. */ uint64_t is_dummy_ddl_for_inner_visibility_: IS_DUMMY_DDL_FOR_INNER_VISIBILITY_BIT; + uint64_t is_major_refreshing_mview_ : IS_MAJOR_REFRESHING_MVIEW_BIT; uint64_t reserved_bit_ : RESERVED_BIT; }; }; diff --git a/deps/oblib/src/lib/ob_define.h b/deps/oblib/src/lib/ob_define.h index 87f7a13cc5..c9729defa2 100644 --- a/deps/oblib/src/lib/ob_define.h +++ b/deps/oblib/src/lib/ob_define.h @@ -648,7 +648,10 @@ const char *const OB_MLOG_ROWID_COLUMN_NAME = "M_ROW$$"; const uint64_t OB_MAX_TMP_COLUMN_ID = OB_ALL_MAX_COLUMN_ID - OB_END_RESERVED_COLUMN_ID_NUM; -const int32_t OB_COUNT_AGG_PD_COLUMN_ID = INT32_MAX - 1; +// pseudo column id used in special table scan +const uint64_t OB_COUNT_AGG_PD_COLUMN_ID = INT32_MAX - 1; +const uint64_t OB_MAJOR_REFRESH_MVIEW_OLD_NEW_COLUMN_ID = INT32_MAX - 2; + const int64_t OB_MAX_AUTOINC_SEQ_VALUE = (1L << 40) - 1; // max value for 40bit const char *const OB_UPDATE_MSG_FMT = "Rows matched: %ld Changed: %ld Warnings: %ld"; @@ -2032,6 +2035,8 @@ const int64_t MAX_CONSTRAINT_NAME_LEN = 128; const char *const OB_LOG_ROW_VALUE_PARTIAL_LOB = "partial_lob"; const char *const OB_LOG_ROW_VALUE_PARTIAL_JSON = "partial_json"; const char *const OB_LOG_ROW_VALUE_PARTIAL_ALL = "partial_all"; +// default duplicate read consistency is strong +const int64_t OB_DEFAULT_DUPLICATE_READ_CONSISTENCY = 0; // json partial update expr flag enum ObJsonPartialUpdateFlag { diff --git a/deps/oblib/src/lib/statistic_event/ob_stat_event.h b/deps/oblib/src/lib/statistic_event/ob_stat_event.h index f388c561c9..3f011191aa 100644 --- a/deps/oblib/src/lib/statistic_event/ob_stat_event.h +++ b/deps/oblib/src/lib/statistic_event/ob_stat_event.h @@ -270,6 +270,8 @@ STAT_EVENT_ADD_DEF(LOG_KV_CACHE_HIT, "log kv cache hit", ObStatClassIds::CACHE, STAT_EVENT_ADD_DEF(LOG_KV_CACHE_MISS, "log kv cache miss", ObStatClassIds::CACHE, 50066, false, true, true) STAT_EVENT_ADD_DEF(DATA_BLOCK_CACHE_MISS, "data block cache miss", ObStatClassIds::CACHE, 50067, true, true, true) STAT_EVENT_ADD_DEF(INDEX_BLOCK_CACHE_MISS, "index block cache miss", ObStatClassIds::CACHE, 50068, true, true, true) +STAT_EVENT_ADD_DEF(MULTI_VERSION_FUSE_ROW_CACHE_HIT, "multi version fuse row cache hit", ObStatClassIds::CACHE, 50069, true, true, true) +STAT_EVENT_ADD_DEF(MULTI_VERSION_FUSE_ROW_CACHE_MISS, "multi version fuse row cache miss", ObStatClassIds::CACHE, 50070, true, true, true) // STORAGE STAT_EVENT_ADD_DEF(MEMSTORE_LOGICAL_READS, "MEMSTORE_LOGICAL_READS", STORAGE, "MEMSTORE_LOGICAL_READS", true, true, false) diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h index 396e3a5971..8aa0f3e0e2 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h @@ -1205,7 +1205,11 @@ PCODE_DEF(OB_KILL_QUERY_CLIENT_SESSION, 0x162E) PCODE_DEF(OB_ENABLE_SS_MICRO_CACHE, 0x162F) PCODE_DEF(OB_GET_SS_MICRO_CACHE_INFO, 0x1630) //PCODE_DEF(OB_MODULE_DATA, 0x1631) + PCODE_DEF(OB_CLEAR_SS_MICRO_CACHE, 0x1632) +PCODE_DEF(OB_COLLECT_MV_MERGE_INFO, 0x1633) +PCODE_DEF(OB_FETCH_STABLE_MEMBER_LIST, 0x1634) + //**** 注意：在此行之前增加新的RPC ID ****** // //占位须知： diff --git a/mittest/env/ob_simple_server_helper.cpp b/mittest/env/ob_simple_server_helper.cpp index 526e814dc0..2b5c353ffa 100644 --- a/mittest/env/ob_simple_server_helper.cpp +++ b/mittest/env/ob_simple_server_helper.cpp @@ -20,6 +20,8 @@ #include "logservice/ob_log_service.h" #include "unittest/storage/init_basic_struct.h" #include "lib/profile/ob_trace_id.h" +#include "storage/tx/ob_trans_service.h" +#include "share/ob_global_stat_proxy.h" namespace oceanbase @@ -33,26 +35,41 @@ int SimpleServerHelper::create_ls(uint64_t tenant_id, ObAddr addr) } int ret = OB_SUCCESS; int64_t affected_rows = 0; - static int64_t start_ls_id = 1001; - ObLSID ls_id(ATOMIC_AAF(&start_ls_id,1)); - if (OB_FAIL(GCTX.sql_proxy_->write(tenant_id, "alter system set enable_rebalance=false", affected_rows))) { - } - if (OB_SUCC(ret)) { + int64_t start_ls_id = 0; + ObLSID ls_id; + if (OB_FAIL(g_select_int64(tenant_id, "select max(ls_id) val from __all_ls", start_ls_id))) { + LOG_WARN("fail to get max ls id", KR(ret)); + } else if (FALSE_IT(ls_id = max(1001, start_ls_id + 1))) { + } else if (OB_FAIL(GCTX.sql_proxy_->write(tenant_id, "alter system set enable_rebalance=false", affected_rows))) { + } else { ObSqlString sql; - sql.assign_fmt("insert into __all_ls (ls_id, ls_group_id, status, flag, create_scn) values(%ld, 1001,'NORMAL', '',0)", ls_id.id()); + sql.assign_fmt("insert into __all_ls (ls_id, ls_group_id, status, flag, create_scn) values(%ld, 1001,'NORMAL', '', 1)", ls_id.id()); if (FAILEDx(GCTX.sql_proxy_->write(tenant_id, sql.ptr(), affected_rows))) { } sql.assign_fmt("insert into __all_ls_status (tenant_id, ls_id, status, ls_group_id, unit_group_id, primary_zone) values(%ld, %ld,'NORMAL', 1001, 1001, 'zone1')", tenant_id, ls_id.id()); if (FAILEDx(GCTX.sql_proxy_->write(gen_meta_tenant_id(tenant_id), sql.ptr(), affected_rows))) { } } + share::SCN merge_scn; if (OB_FAIL(ret)) { return ret; + } else { + ObGlobalStatProxy proxy(*GCTX.sql_proxy_, tenant_id); + int tmp_ret = proxy.get_major_refresh_mv_merge_scn(false, merge_scn); + if (tmp_ret != OB_SUCCESS) { + LOG_WARN("get merge_scn failed", KR(tmp_ret)); + } + } + if (!merge_scn.is_valid()) { + merge_scn.set_min(); } MTL_SWITCH(tenant_id) { ObCreateLSArg arg; ObLSService* ls_svr = MTL(ObLSService*); FR(gen_create_ls_arg(tenant_id, ls_id, arg)); + arg.major_mv_merge_info_.major_mv_merge_scn_publish_.atomic_store(merge_scn); + arg.major_mv_merge_info_.major_mv_merge_scn_safe_calc_.atomic_store(merge_scn); + arg.major_mv_merge_info_.major_mv_merge_scn_.atomic_store(merge_scn); FR(ls_svr->create_ls(arg)); LOG_INFO("set member list"); ObLSHandle handle; @@ -914,4 +931,24 @@ void InjectTxFaultHelper::release() tx_injects_.clear(); } +int SimpleServerHelper::get_tx_desc(uint64_t tenant_id, ObTransID tx_id, ObTxDesc *&tx_desc) +{ + int ret = OB_SUCCESS; + MTL_SWITCH(tenant_id) { + ret = MTL(ObTransService*)->tx_desc_mgr_.get(tx_id, tx_desc); + } + return ret; +} + +int SimpleServerHelper::revert_tx_desc(uint64_t tenant_id, ObTxDesc *tx_desc) +{ + int ret = OB_SUCCESS; + MTL_SWITCH(tenant_id) { + if (OB_NOT_NULL(tx_desc)) { + MTL(ObTransService*)->tx_desc_mgr_.revert(*tx_desc); + } + } + return ret; +} + } diff --git a/mittest/env/ob_simple_server_helper.h b/mittest/env/ob_simple_server_helper.h index 45a463b81c..5f20c6ec3f 100644 --- a/mittest/env/ob_simple_server_helper.h +++ b/mittest/env/ob_simple_server_helper.h @@ -81,6 +81,8 @@ public: static int wait_weak_read_ts_advance(uint64_t tenant_id, ObLSID ls_id1, ObLSID ls_id2); static int enable_wrs(uint64_t tenant_id, ObLSID ls_id, bool enable); static int modify_wrs(uint64_t tenant_id, ObLSID ls_id, int64_t add_ns = 10 * 1000 * 1000 * 1000L); + static int get_tx_desc(uint64_t tenant_id, ObTransID tx_id, ObTxDesc *&tx_desc); + static int revert_tx_desc(uint64_t tenant_id, ObTxDesc *tx_desc); }; class InjectTxFaultHelper : public transaction::ObLSTxLogAdapter diff --git a/mittest/simple_server/CMakeLists.txt b/mittest/simple_server/CMakeLists.txt index 19ecb12306..7f2ff033f2 100644 --- a/mittest/simple_server/CMakeLists.txt +++ b/mittest/simple_server/CMakeLists.txt @@ -35,6 +35,11 @@ function(ob_freeze_observer case) target_link_libraries(${case} PRIVATE gtest gmock observer_test oceanbase) endfunction() +function(ob_farm_observer case) + ob_unittest(${ARGV}) + target_link_libraries(${case} PRIVATE gtest gmock observer_test oceanbase) +endfunction() + function(ob_offline_observer case case_file) add_executable(${case} EXCLUDE_FROM_ALL @@ -60,6 +65,7 @@ endfunction() ob_offline_observer(test_simple_ob test_ob_simple_cluster.cpp) ob_offline_observer(test_transfer_tx test_transfer_tx.cpp) ob_offline_observer(test_tx_data test_tx_data.cpp) +ob_offline_observer(test_collect_mv test_collect_mv.cpp) ob_unittest_observer(test_transfer_no_kill_tx test_transfer_tx.cpp) ob_unittest_observer(test_standby_balance test_standby_balance_ls_group.cpp) @@ -142,3 +148,5 @@ ob_ha_unittest_observer(test_transfer_complete_restart_with_mds_flush storage_ha ob_ha_unittest_observer(test_transfer_with_empty_shell storage_ha/test_transfer_with_empty_shell.cpp) ob_ha_unittest_observer(test_mds_transaction test_mds_transaction.cpp) errsim_ha_unittest_observer(errsim_test_transfer_handler errsim/storage_ha/errsim_test_transfer_handler.cpp) + +ob_farm_observer(test_collect_mv_farm test_collect_mv.cpp) diff --git a/mittest/simple_server/env/ob_simple_server.cpp b/mittest/simple_server/env/ob_simple_server.cpp index aa695c8efd..3945c23ad4 100644 --- a/mittest/simple_server/env/ob_simple_server.cpp +++ b/mittest/simple_server/env/ob_simple_server.cpp @@ -158,6 +158,9 @@ int ObSimpleServer::simple_init() system(("rm -rf " + run_dir_).c_str()); system(("rm -f *born*log*")); system(("rm -f *restart*log*")); + + //check admin sys package exist + DIR *admin_dir = opendir("admin"); SERVER_LOG(INFO, "create dir and change work dir start.", K(run_dir_.c_str())); if (OB_FAIL(mkdir(run_dir_.c_str(), 0777))) { } else if (OB_FAIL(chdir(run_dir_.c_str()))) { @@ -173,6 +176,7 @@ int ObSimpleServer::simple_init() dirs.push_back("etc"); dirs.push_back("log"); dirs.push_back("wallet"); + dirs.push_back("admin"); dirs.push_back(data_dir + "/clog"); dirs.push_back(data_dir + "/slog"); @@ -184,6 +188,9 @@ int ObSimpleServer::simple_init() return ret; } } + if (admin_dir != NULL) { + system(("cp ../admin/* admin/")); + } std::string port_file_name = std::string("port.txt"); FILE *outfile = nullptr; diff --git a/mittest/simple_server/test_collect_mv.cpp b/mittest/simple_server/test_collect_mv.cpp new file mode 100644 index 0000000000..c71f4e3baa --- /dev/null +++ b/mittest/simple_server/test_collect_mv.cpp @@ -0,0 +1,679 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#define USING_LOG_PREFIX SERVER +#define protected public +#define private public + +#include "env/ob_simple_cluster_test_base.h" +#include "lib/mysqlclient/ob_mysql_result.h" +#include "storage/init_basic_struct.h" +#include "storage/tx_storage/ob_ls_service.h" +#include "rootserver/ob_tenant_balance_service.h" +#include "storage/tx/ob_trans_part_ctx.h" +#include "share/balance/ob_balance_job_table_operator.h" +#include "storage/tablelock/ob_table_lock_service.h" +#include "rootserver/ob_balance_group_ls_stat_operator.h" +#include "storage/tablet/ob_tablet.h" +#include "logservice/ob_log_service.h" +#include "mittest/env/ob_simple_server_helper.h" +#include "storage/concurrency_control/ob_multi_version_garbage_collector.h" +#include "rootserver/mview/ob_mview_maintenance_service.h" +#include "storage/compaction/ob_tenant_tablet_scheduler.h" + +namespace oceanbase +{ +namespace unittest +{ + +using namespace oceanbase::transaction; +using namespace oceanbase::storage; +using namespace oceanbase::rootserver; +using namespace oceanbase::compaction; + +#define EQ(x, y) GTEST_ASSERT_EQ(x, y); +#define NE(x, y) GTEST_ASSERT_NE(x, y); +#define LE(x, y) GTEST_ASSERT_LE(x, y); +#define GE(x, y) GTEST_ASSERT_GE(x, y); +#define GT(x, y) GTEST_ASSERT_GT(x, y); +#define LT(x, y) GTEST_ASSERT_LT(x, y); + +class TestRunCtx +{ +public: + uint64_t tenant_id_ = 0; + int64_t time_sec_ = 0; + int64_t start_time_ = ObTimeUtil::current_time(); + bool stop_ = false; + std::thread th_; + std::thread worker_; +}; + +TestRunCtx R; + +class ObCollectMV : public ObSimpleClusterTestBase +{ +public: + // 指定case运行目录前缀 test_ob_simple_cluster_ + ObCollectMV() : ObSimpleClusterTestBase("test_collect_mv_", "20G", "20G") {} + + int wait_major_mv_refresh_finish(uint64_t scn); + int wait_acquired_snapshot_advance(uint64_t scn); + int wait_old_sstable_gc_end(uint64_t scn); + void process(); + int do_balance_inner_(uint64_t tenant_id); +private: +}; + +int ObCollectMV::wait_major_mv_refresh_finish(uint64_t scn) +{ + int ret = OB_SUCCESS; + + ObSqlString sql_string; + sql_string.assign_fmt("select min(last_refresh_scn) val from oceanbase.__all_mview where refresh_mode=4"); + uint64_t val = 0; + while (OB_SUCC(ret)) { + if (OB_FAIL(SSH::g_select_uint64(R.tenant_id_, sql_string.ptr(), val))) { + } else if (val >= scn) { + break; + } else { + LOG_INFO("wait major mv refresh", K(scn), K(val)); + ::sleep(2); + } + } + return ret; +} + +int ObCollectMV::wait_acquired_snapshot_advance(uint64_t scn) +{ + int ret = OB_SUCCESS; + ObSqlString sql_string; + sql_string.assign_fmt("select min(snapshot_scn) val from oceanbase.__all_acquired_snapshot where snapshot_type=5"); + uint64_t val = 0; + while (OB_SUCC(ret)) { + if (OB_FAIL(SSH::g_select_uint64(R.tenant_id_, sql_string.ptr(), val))) { + } else if (val >= scn) { + break; + } else { + LOG_INFO("wait acquired_snapshot_advance", K(scn), K(val)); + MTL_SWITCH(R.tenant_id_) { + storage::ObTenantFreezeInfoMgr *mgr = MTL(storage::ObTenantFreezeInfoMgr *); + const int64_t snapshot_for_tx = mgr->get_min_reserved_snapshot_for_tx(); + LOGI("wait_acquired_snapshot_advance: %ld %ld %ld", scn, snapshot_for_tx,val); + } + ::sleep(2); + } + } + return ret; +} + +int ObCollectMV::wait_old_sstable_gc_end(uint64_t scn) +{ + int ret = OB_SUCCESS; + ObSqlString sql_string; + sql_string.assign_fmt("select min(end_log_scn) val from oceanbase.__all_virtual_table_mgr where table_type=10 and tablet_id>200000"); + uint64_t val = 0; + while (OB_SUCC(ret)) { + if (OB_FAIL(SSH::g_select_uint64(R.tenant_id_, sql_string.ptr(), val))) { + } else if (val >= scn) { + break; + } else { + LOG_INFO("wait old sstable gc", K(scn), K(val)); + ::sleep(2); + } + } + return ret; +} + +void ObCollectMV::process() +{ + int ret = OB_SUCCESS; + MTL_SWITCH(R.tenant_id_) { + ObMViewMaintenanceService *mv_service = MTL(ObMViewMaintenanceService*); + mv_service->mview_push_refresh_scn_task_.runTimerTask(); + mv_service->mview_push_snapshot_task_.runTimerTask(); + mv_service->replica_safe_check_task_.runTimerTask(); + mv_service->collect_mv_merge_info_task_.runTimerTask(); + mv_service->mview_clean_snapshot_task_.runTimerTask(); + MTL(ObTenantTabletScheduler*)->timer_task_mgr_.sstable_gc_task_.runTimerTask(); + } +} + +int ObCollectMV::do_balance_inner_(uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + static std::mutex mutex; + mutex.lock(); + MTL_SWITCH(tenant_id) { + LOG_INFO("worker to do partition_balance"); + auto b_svr = MTL(rootserver::ObTenantBalanceService*); + b_svr->reset(); + b_svr->stop(); + int64_t job_cnt = 0; + int64_t start_time = OB_INVALID_TIMESTAMP, finish_time = OB_INVALID_TIMESTAMP; + ObBalanceJob job; + if (OB_FAIL(b_svr->gather_stat_())) { + LOG_WARN("failed to gather stat", KR(ret)); + } else if (OB_FAIL(b_svr->gather_ls_status_stat(tenant_id, b_svr->ls_array_))) { + LOG_WARN("failed to gather stat", KR(ret)); + } else if (OB_FAIL(ObBalanceJobTableOperator::get_balance_job( + tenant_id, false, *GCTX.sql_proxy_, job, start_time, finish_time))) { + if (OB_ENTRY_NOT_EXIST == ret) { + //NO JOB, need check current ls status + ret = OB_SUCCESS; + job_cnt = 0; + } else { + LOG_WARN("failed to get balance job", KR(ret), K(tenant_id)); + } + } else if (OB_FAIL(b_svr->try_finish_current_job_(job, job_cnt))) { + LOG_WARN("failed to finish current job", KR(ret), K(job)); + } + if (OB_SUCC(ret) && job_cnt == 0 && OB_FAIL(b_svr->partition_balance_(true))) { + LOG_WARN("failed to do partition balance", KR(ret)); + } + } + mutex.unlock(); + return ret; + +} + + +TEST_F(ObCollectMV, prepare) +{ + int ret = OB_SUCCESS; + LOGI("observer start"); + R.th_ = std::thread([]() { + while (!R.stop_) { + fflush(stdout); + ::usleep(100 * 1000); + }}); + concurrency_control::ObMultiVersionGarbageCollector::GARBAGE_COLLECT_EXEC_INTERVAL = 1_s; + concurrency_control::ObMultiVersionGarbageCollector::GARBAGE_COLLECT_RETRY_INTERVAL = 3_s; + concurrency_control::ObMultiVersionGarbageCollector::GARBAGE_COLLECT_RECLAIM_DURATION = 3_s; + + LOGI("create tenant begin"); + // 创建普通租户tt1 + EQ(OB_SUCCESS, create_tenant("tt1", "12G", "16G", false, 10)); + // 获取租户tt1的tenant_id + EQ(OB_SUCCESS, get_tenant_id(R.tenant_id_)); + NE(0, R.tenant_id_); + // 初始化普通租户tt1的sql proxy + EQ(OB_SUCCESS, get_curr_simple_server().init_sql_proxy2()); + int tenant_id = R.tenant_id_; + LOGI("create tenant finish"); + + // 在单节点ObServer下创建新的日志流, 注意避免被RS任务GC掉 + EQ(0, SSH::create_ls(R.tenant_id_, get_curr_observer().self_addr_)); + int64_t ls_count = 0; + EQ(0, SSH::g_select_int64(R.tenant_id_, "select count(ls_id) as val from oceanbase.__all_ls where ls_id!=1", ls_count)); + EQ(2, ls_count); + + int64_t affected_rows; + EQ(0, GCTX.sql_proxy_->write("alter system set _enable_parallel_table_creation = false tenant=all", affected_rows)); + EQ(0, GCTX.sql_proxy_->write("alter system set undo_retention='0' tenant=all", affected_rows)); + EQ(0, GCTX.sql_proxy_->write("alter system set ob_compaction_schedule_interval='5s' tenant=all", affected_rows)); + EQ(0, GCTX.sql_proxy_->write("alter system set merger_check_interval='10s' tenant=all", affected_rows)); + + MTL_SWITCH(R.tenant_id_) { + TG_STOP(MTL(ObTenantTabletScheduler *)->timer_task_mgr_.sstable_gc_tg_id_); + } + + R.worker_ = std::thread([this]() { + while (!R.stop_) { + process(); + do_balance_inner_(R.tenant_id_); + ::sleep(3); + }}); +} + +TEST_F(ObCollectMV, read) +{ + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); + int64_t affected_rows; + int ret = OB_SUCCESS; + EQ(0, sql_proxy.write("create table t(c1 int)", affected_rows)); + + sqlclient::ObISQLConnection *conn = NULL; + EQ(0, sql_proxy.acquire(conn)); + EQ(0, SSH::write(conn, "set autocommit=0")); + EQ(0, SSH::write(conn, "SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ")); + EQ(0, SSH::write(conn, "set ob_query_timeout=99900000000")); + EQ(0, SSH::write(conn, "set ob_trx_timeout=1000000000")); + int64_t val = 0; + EQ(0, SSH::select_int64(conn, "select count(*) val from t", val)); + ObString trace_id; + ObTransID tx_id; + int64_t session_id = -1; + EQ(0, SSH::find_session(conn, session_id)); + EQ(0, SSH::find_trace_id(conn, trace_id)); + EQ(0, SSH::find_tx(conn, tx_id)); + ObTxDesc *tx_desc = NULL; + EQ(0, SSH::get_tx_desc(R.tenant_id_, tx_id, tx_desc)); + uint64_t read_snapshot = tx_desc->snapshot_version_.get_val_for_gts(); + int tx_state = (int)tx_desc->state_; + EQ(0, SSH::revert_tx_desc(R.tenant_id_, tx_desc)); + GT(read_snapshot, 0); + LOGI("session_id:%ld tx_id:%ld read_snapshot:%ld %d", session_id, tx_id.tx_id_,read_snapshot, tx_state); + bool wait_end = false; + while (OB_SUCC(ret) && !wait_end) { + ::sleep(2); + MTL_SWITCH(R.tenant_id_) { + storage::ObTenantFreezeInfoMgr *mgr = MTL(storage::ObTenantFreezeInfoMgr *); + const int64_t snapshot_for_tx = mgr->get_min_reserved_snapshot_for_tx(); + LOGI("snapshot_for_tx:%ld", snapshot_for_tx); + if (snapshot_for_tx == read_snapshot) { + wait_end = true; + } else if (snapshot_for_tx > read_snapshot) { + ret = OB_ERR_UNEXPECTED; + LOGE("snapshot_for_tx big than read_snapshot: %ld %ld", snapshot_for_tx, read_snapshot); + } + } + } + EQ(OB_SUCCESS, ret); + EQ(0, SSH::select_int64(conn, "select count(*) val from t", val)); + EQ(0, conn->commit()); + EQ(0, sql_proxy.close(conn, true)); +} + +TEST_F(ObCollectMV, basic) +{ + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); + int64_t affected_rows = 0; + LOGI("create tg"); + EQ(0, sql_proxy.write("create tablegroup tg1 sharding='PARTITION'", affected_rows)); + LOGI("create table"); + EQ(0, sql_proxy.write("create table t1(c1 int,c2 int, primary key(c1,c2)) tablegroup='tg1' partition by hash(c1) partitions 2", affected_rows)); + EQ(0, sql_proxy.write("create table t2(c2 int, c3 int, primary key(c2)) duplicate_scope = 'cluster' duplicate_read_consistency='weak'", affected_rows)); + LOGI("create mview"); + EQ(0, sql_proxy.write( + "create materialized view compact_mv_1 (primary key(t1_c1,t1_c2)) tablegroup='tg1' " + "partition by hash(t1_c1) partitions 2 " + "REFRESH FAST ON DEMAND ENABLE QUERY REWRITE ENABLE ON QUERY COMPUTATION " + "as select /*+read_consistency(weak) use_nl(t1 t2) leading(t1 t2) " + "use_das(t2) no_use_nl_materialization(t2)*/ t1.c1 t1_c1,t1.c2 t1_c2,t2.c2 t2_c2,t2.c3 t2_c3 " + "from t1 join t2 on t1.c2=t2.c2", + affected_rows)); + LOGI("check refresh_mode"); + int64_t refresh_mode = 0; + EQ(0, SSH::g_select_int64(R.tenant_id_, "select refresh_mode val from oceanbase.__all_mview t1, oceanbase.__all_table t2 where t1.mview_id=t2.table_id and t2.table_name='compact_mv_1'", refresh_mode)); + EQ(4, refresh_mode); + LOGI("get refresh_scn"); + uint64_t refresh_scn = 0; + EQ(0, SSH::g_select_uint64(R.tenant_id_, "select last_refresh_scn val from oceanbase.__all_mview t1, oceanbase.__all_table t2 where t1.mview_id=t2.table_id and t2.table_name='compact_mv_1'", refresh_scn)); + GT(refresh_scn, 0); + + ObSqlString sql_string; + int64_t now = ObTimeUtil::current_time_ns(); + sql_string.assign_fmt("select /*+no_mv_rewrite*/ count(*) val from compact_mv_1 as of snapshot %ld", now); + int64_t val = 0; + EQ(0, SSH::select_int64(sql_proxy, sql_string.ptr(), val)); + + LOGI("start major freeze>>>"); + EQ(0, sql_proxy.write("alter system major freeze", affected_rows)); + uint64_t new_freeze_scn = 0; + EQ(0, SSH::g_select_uint64(R.tenant_id_, "select max(frozen_scn) val from oceanbase.__all_freeze_info", new_freeze_scn)); + GT(new_freeze_scn, refresh_scn); + + LOGI("wait major mv refresh finish>>>"); + EQ(0, wait_major_mv_refresh_finish(new_freeze_scn)); + LOGI("wait acquired snapshot advance>>>"); + EQ(0, wait_acquired_snapshot_advance(new_freeze_scn)); + LOGI("wait old sstable gc>>>"); + EQ(0, wait_old_sstable_gc_end(new_freeze_scn)); + + NE(0, SSH::select_int64(sql_proxy, sql_string.ptr(), affected_rows)); + sql_string.assign_fmt("select /*+no_mv_rewrite*/ count(*) val from compact_mv_1 as of snapshot %ld", new_freeze_scn); + EQ(0, SSH::select_int64(sql_proxy, sql_string.ptr(), val)); +} + +TEST_F(ObCollectMV, read_safe) +{ + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); + int64_t affected_rows; + int ret = OB_SUCCESS; + + sqlclient::ObISQLConnection *conn = NULL; + EQ(0, sql_proxy.acquire(conn)); + EQ(0, SSH::write(conn, "set autocommit=0")); + EQ(0, SSH::write(conn, "SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ")); + EQ(0, SSH::write(conn, "set ob_query_timeout=99900000000")); + EQ(0, SSH::write(conn, "set ob_trx_timeout=1000000000")); + int64_t val = 0; + EQ(0, SSH::select_int64(conn, "select count(*) val from compact_mv_1", val)); + ObString trace_id; + ObTransID tx_id; + EQ(0, SSH::find_trace_id(conn, trace_id)); + EQ(0, SSH::find_tx(conn, tx_id)); + ObTxDesc *tx_desc = NULL; + EQ(0, SSH::get_tx_desc(R.tenant_id_, tx_id, tx_desc)); + uint64_t read_snapshot = tx_desc->snapshot_version_.get_val_for_gts(); + EQ(0, SSH::revert_tx_desc(R.tenant_id_, tx_desc)); + LOGI("read trans stmt trace_id: %s tx_id: %ld snapshot: %ld", trace_id.ptr(), tx_id.tx_id_, read_snapshot); + + LOGI("start major freeze >>>"); + EQ(0, sql_proxy.write("alter system major freeze", affected_rows)); + uint64_t new_freeze_scn = 0; + EQ(0, SSH::g_select_uint64(R.tenant_id_, "select max(frozen_scn) val from oceanbase.__all_freeze_info", new_freeze_scn)); + LOGI("wait major mv refresh finish >>>"); + EQ(0, wait_major_mv_refresh_finish(new_freeze_scn)); + + LOGI("read with old read_snapshot"); + EQ(0, SSH::select_int64(conn, "select count(*) val from compact_mv_1", val)); + EQ(0, SSH::find_trace_id(conn, trace_id)); + LOGI("read trans stmt trace_id: %s", trace_id.ptr()); + ::sleep(30); + + int64_t snapshot_for_tx = 0; + MTL_SWITCH(R.tenant_id_) { + storage::ObTenantFreezeInfoMgr *mgr = MTL(storage::ObTenantFreezeInfoMgr *); + snapshot_for_tx = mgr->get_min_reserved_snapshot_for_tx(); + } + EQ(OB_SUCCESS, ret); + LOGI("read with old read_snapshot snapshot_for_tx:%ld read_snapshot:%ld", snapshot_for_tx, read_snapshot); + LE(snapshot_for_tx, read_snapshot); + EQ(0, SSH::select_int64(conn, "select count(*) val from compact_mv_1", val)); + EQ(0, SSH::find_trace_id(conn, trace_id)); + LOGI("read trans stmt trace_id: %s", trace_id.ptr()); + EQ(0, conn->commit()); + EQ(0, sql_proxy.close(conn, true)); + LOGI("wait old sstable gc>>>"); + EQ(0, wait_old_sstable_gc_end(new_freeze_scn)); +} + +TEST_F(ObCollectMV, snapshot_gc) +{ + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); + ObSqlString sql_string; + int64_t affected_rows = 0; + int64_t base_tablet_id = 0; + int64_t cnt = 0; + int64_t snapshot_val = 0; + + LOGI("create tg"); + EQ(0, sql_proxy.write("create tablegroup gc_tg1 sharding='PARTITION'", affected_rows)); + LOGI("create table"); + EQ(0, sql_proxy.write("create table gc_t1(c1 int,c2 int, primary key(c1,c2)) tablegroup='gc_tg1' " + "partition by hash(c1) partitions 2", + affected_rows)); + EQ(0, sql_proxy.write("create table gc_t2(c2 int, c3 int, primary key(c2)) duplicate_scope = " + "'cluster' duplicate_read_consistency='weak'", + affected_rows)); + EQ(0, SSH::select_int64( + sql_proxy, "select t2.tablet_id val from oceanbase.__all_table t1 join oceanbase.__all_tablet_to_ls t2 on t1.table_id = t2.table_id where t1.table_name = 'gc_t1' limit 1", + base_tablet_id)); + + LOGI("create mview1"); + int64_t mv1_tablet_id = 0; + int64_t snapshot_scn_of_base_by_mv1 = 0; + EQ(0, sql_proxy.write( + "create materialized view gc_compact_mv_1 (primary key(t1_c1,t1_c2)) tablegroup='gc_tg1' " + "partition by hash(t1_c1) partitions 2 " + "REFRESH FAST ON DEMAND ENABLE QUERY REWRITE ENABLE ON QUERY COMPUTATION " + "as select /*+read_consistency(weak) use_nl(t1 t2) leading(t1 t2) " + "use_das(t2) no_use_nl_materialization(t2)*/ t1.c1 t1_c1,t1.c2 t1_c2,t2.c2 t2_c2,t2.c3 t2_c3 " + "from gc_t1 t1 join gc_t2 t2 on t1.c2=t2.c2", + affected_rows)); + EQ(0, SSH::select_int64( + sql_proxy, + "select t3.tablet_id val from oceanbase.__all_table t1, oceanbase.__all_table t2, oceanbase.__all_tablet_to_ls t3 where " + "t1.table_id = t2.data_table_id and t2.table_name='gc_compact_mv_1' and t1.table_id = t3.table_id limit 1", + mv1_tablet_id)); + sql_string.assign_fmt( + "select count(*) val from oceanbase.__all_acquired_snapshot where tablet_id = %ld", + mv1_tablet_id); + EQ(0, SSH::select_int64(sql_proxy, sql_string.ptr(), cnt)); + EQ(1, cnt); + sql_string.assign_fmt( + "select snapshot_scn val from oceanbase.__all_acquired_snapshot where tablet_id = %ld", + base_tablet_id); + EQ(0, SSH::select_int64(sql_proxy, sql_string.ptr(), snapshot_scn_of_base_by_mv1)); + + LOGI("create mview2"); + int64_t mv2_tablet_id = 0; + EQ(0, sql_proxy.write( + "create materialized view gc_compact_mv_2 (primary key(t1_c1,t1_c2)) tablegroup='gc_tg1' " + "partition by " + "hash(t1_c1) partitions 2 REFRESH FAST ON DEMAND ENABLE QUERY REWRITE ENABLE ON QUERY " + "COMPUTATION as select /*+read_consistency(weak) use_nl(t1 t2) leading(t1 t2) " + "use_das(t2) no_use_nl_materialization(t2)*/ t1.c1 t1_c1,t1.c2 t1_c2,t2.c2 t2_c2,t2.c3 t2_c3 " + "from gc_t1 t1 join gc_t2 t2 on t1.c2=t2.c2", + affected_rows)); + EQ(0, SSH::select_int64( + sql_proxy, + "select t3.tablet_id val from oceanbase.__all_table t1, oceanbase.__all_table t2, oceanbase.__all_tablet_to_ls t3 where " + "t1.table_id = t2.data_table_id and t2.table_name='gc_compact_mv_2' and t1.table_id = t3.table_id limit 1", + mv2_tablet_id)); + sql_string.assign_fmt( + "select count(*) val from oceanbase.__all_acquired_snapshot where tablet_id = %ld", mv2_tablet_id); + EQ(0, SSH::select_int64(sql_proxy, sql_string.ptr(), cnt)); + EQ(1, cnt); + + LOGI("create mview3"); + int64_t mv3_tablet_id = 0; + EQ(0, sql_proxy.write( + "create materialized view gc_compact_mv_3 (primary key(t1_c1,t1_c2)) tablegroup='gc_tg1' " + "partition by hash(t1_c1) partitions 2 " + "REFRESH FAST ON DEMAND ENABLE QUERY REWRITE ENABLE ON QUERY COMPUTATION " + "as select /*+read_consistency(weak) use_nl(t1 t2) leading(t1 t2) " + "use_das(t2) no_use_nl_materialization(t2)*/ t1.c1 t1_c1,t1.c2 t1_c2,t2.c2 t2_c2,t2.c3 t2_c3 " + "from gc_t1 t1 join gc_t2 t2 on t1.c2=t2.c2", + affected_rows)); + EQ(0, SSH::select_int64( + sql_proxy, + "select t3.tablet_id val from oceanbase.__all_table t1, oceanbase.__all_table t2, oceanbase.__all_tablet_to_ls t3 where " + "t1.table_id = t2.data_table_id and t2.table_name='gc_compact_mv_3' and t1.table_id = t3.table_id limit 1", + mv3_tablet_id)); + sql_string.assign_fmt( + "select count(*) val from oceanbase.__all_acquired_snapshot where tablet_id = %ld", mv3_tablet_id); + EQ(0, SSH::select_int64(sql_proxy, sql_string.ptr(), cnt)); + EQ(1, cnt); + + while (true) { + sql_string.assign_fmt( + "select count(*) val from oceanbase.__all_acquired_snapshot where tablet_id = %ld", + base_tablet_id); + EQ(0, SSH::select_int64(sql_proxy, sql_string.ptr(), cnt)); + if (cnt == 1) { + break; + } + ::sleep(2); + LOGI("wait remove redundant snapshot"); + } + sql_string.assign_fmt( + "select snapshot_scn val from oceanbase.__all_acquired_snapshot where tablet_id = %ld", + base_tablet_id); + EQ(0, SSH::select_int64(sql_proxy, sql_string.ptr(), snapshot_val)); + // 清除冗余的快照后，留下的应该是最小的快照 + EQ(snapshot_scn_of_base_by_mv1, snapshot_val); + + LOGI("remove mview1"); + EQ(0, sql_proxy.write("drop materialized view gc_compact_mv_1", affected_rows)); + while (true) { + sql_string.assign_fmt( + "select count(*) val from oceanbase.__all_acquired_snapshot where tablet_id = %ld", + mv1_tablet_id); + EQ(0, SSH::select_int64(sql_proxy, sql_string.ptr(), cnt)); + if (cnt == 0) { + break; + } + ::sleep(2); + LOGI("wait remove snapshot of mview1"); + } + sql_string.assign_fmt( + "select count(*) val from oceanbase.__all_acquired_snapshot where tablet_id = %ld", + base_tablet_id); + EQ(0, SSH::select_int64(sql_proxy, sql_string.ptr(), cnt)); + // 基表的快照还不应该被回收掉 + EQ(1, cnt); + + LOGI("remove mview2 and mview3"); + EQ(0, sql_proxy.write("drop materialized view gc_compact_mv_2", affected_rows)); + EQ(0, sql_proxy.write("drop materialized view gc_compact_mv_3", affected_rows)); + while (true) { + sql_string.assign_fmt("select count(*) val from oceanbase.__all_acquired_snapshot where " + "tablet_id = %ld or tablet_id = %ld", + mv2_tablet_id, mv3_tablet_id); + EQ(0, SSH::select_int64(sql_proxy, sql_string.ptr(), cnt)); + if (cnt == 0) { + break; + } + ::sleep(2); + LOGI("wait remove snapshot of mview2 and mview3"); + } + sql_string.assign_fmt( + "select count(*) val from oceanbase.__all_acquired_snapshot where tablet_id = %ld", + base_tablet_id); + EQ(0, SSH::select_int64(sql_proxy, sql_string.ptr(), cnt)); + // 基表的快照应该被回收掉了 + EQ(0, cnt); + + LOGI("clean up"); + EQ(0, sql_proxy.write("drop table gc_t1", affected_rows)); + EQ(0, sql_proxy.write("drop table gc_t2", affected_rows)); +} + +TEST_F(ObCollectMV, tablegroup) +{ + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); + int64_t affected_rows = 0; + + EQ(0, sql_proxy.write("create table t100(c1 int) partition by hash(c1) partitions 2", affected_rows)); + EQ(0, sql_proxy.write("create tablegroup tg100 sharding='PARTITION'", affected_rows)); + EQ(0, sql_proxy.write("create materialized view mv100 partition by hash(c1) partitions 2 as select * from t100", affected_rows)); + + EQ(0, sql_proxy.write("alter table t100 tablegroup='tg100'", affected_rows)); + EQ(0, sql_proxy.write("alter table mv100 tablegroup='tg100'", affected_rows)); + + int64_t val = -2; + EQ(0, SSH::select_int64(sql_proxy, "select tablegroup_id val from oceanbase.__all_table where table_name='t100'", val)); + GT(val, 0); + // mview + val = -2; + EQ(0, SSH::select_int64(sql_proxy, "select tablegroup_id val from oceanbase.__all_table where table_name='mv100'", val)); + EQ(val, -1); + //mview container + val = -2; + EQ(0, SSH::select_int64(sql_proxy, "select tablegroup_id val from oceanbase.__all_table where table_id in ( \ + select data_table_id from oceanbase.__all_table where table_name='mv100')", val)); + GT(val, 0); + + EQ(0, sql_proxy.write("alter table mv100 tablegroup=''", affected_rows)); + EQ(0, sql_proxy.write("alter table t100 tablegroup=''", affected_rows)); + + EQ(0, sql_proxy.write("drop tablegroup tg100", affected_rows)); + EQ(0, sql_proxy.write("drop table t100", affected_rows)); + EQ(0, sql_proxy.write("drop materialized view mv100", affected_rows)); +} + +TEST_F(ObCollectMV, new_ls) +{ + int ret = OB_SUCCESS; + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); + int64_t val = 0; + EQ(0, SSH::select_int64(sql_proxy, "select column_value val from oceanbase.__all_core_table where column_name='major_refresh_mv_merge_scn'", val)); + GT(val, 0); + int64_t merge_scn = val; + + EQ(0, SSH::create_ls(R.tenant_id_, get_curr_observer().self_addr_)); + + EQ(0, SSH::select_int64(sql_proxy, "select max(ls_id) val from oceanbase.__all_ls", val)); + ObLSID ls_id(val); + + ObLSMeta ls_meta; + MTL_SWITCH(R.tenant_id_) { + ObLSHandle ls_handle; + if (OB_FAIL(MTL(ObLSService*)->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { + } else if (OB_FAIL(ls_handle.get_ls()->get_ls_meta(ls_meta))) { + } + } + EQ(0, ret); + EQ(ls_meta.major_mv_merge_info_.major_mv_merge_scn_.get_val_for_gts(), merge_scn); +} + +TEST_F(ObCollectMV, mv_transfer) +{ + int ret = OB_SUCCESS; + int64_t affected_rows = 0; + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); + + EQ(0, sql_proxy.write("create table tf_t1(c1 int,c2 int, primary key(c1,c2)) " + "partition by hash(c1) partitions 2", + affected_rows)); + EQ(0, sql_proxy.write("create table tf_t2(c2 int, c3 int, primary key(c2)) duplicate_scope = " + "'cluster' duplicate_read_consistency='weak'", + affected_rows)); + EQ(0, sql_proxy.write( + "create materialized view tf_compact_mv_1 (primary key(t1_c1,t1_c2)) tablegroup='tg1' " + "partition by hash(t1_c1) partitions 2 " + "REFRESH FAST ON DEMAND ENABLE QUERY REWRITE ENABLE ON QUERY COMPUTATION " + "as select /*+read_consistency(weak) use_nl(t1 t2) leading(t1 t2) " + "use_das(t2) no_use_nl_materialization(t2)*/ t1.c1 t1_c1,t1.c2 t1_c2,t2.c2 t2_c2,t2.c3 " + "t2_c3 " + "from tf_t1 t1 join tf_t2 t2 on t1.c2=t2.c2", + affected_rows)); + EQ(0, sql_proxy.write("create tablegroup tg200 sharding='NONE'", affected_rows)); + EQ(0, sql_proxy.write("alter table tf_compact_mv_1 tablegroup='tg200'", affected_rows)); + int64_t val = -1; + while (true) { + EQ(0, SSH::select_int64(sql_proxy, "select count(distinct ls_id) val from oceanbase.__all_tablet_to_ls where table_id in (select data_table_id from oceanbase.__all_table where table_name='tf_compact_mv_1')", val)); + if (val == 1) { + break; + } else { + LOGI("wait transfer finish:%ld",val); + ::sleep(1); + } + } +} + +TEST_F(ObCollectMV, end) +{ + int64_t wait_us = R.time_sec_ * 1000 * 1000; + while (ObTimeUtil::current_time() - R.start_time_ < wait_us) { + ob_usleep(1000 * 1000); + } + R.stop_ = true; + R.th_.join(); + R.worker_.join(); +} + +} // end unittest +} // end oceanbase + + +int main(int argc, char **argv) +{ + int c = 0; + int64_t time_sec = 0; + char *log_level = (char*)"INFO"; + while(EOF != (c = getopt(argc,argv,"t:l:"))) { + switch(c) { + case 't': + time_sec = atoi(optarg); + break; + case 'l': + log_level = optarg; + oceanbase::unittest::ObSimpleClusterTestBase::enable_env_warn_log_ = false; + break; + default: + break; + } + } + oceanbase::unittest::init_log_and_gtest(argc, argv); + OB_LOGGER.set_log_level(log_level); + + LOG_INFO("main>>>"); + + oceanbase::unittest::R.time_sec_ = time_sec; + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/mittest/simple_server/test_ls_recover.cpp b/mittest/simple_server/test_ls_recover.cpp index 7801d88d63..a42c075274 100644 --- a/mittest/simple_server/test_ls_recover.cpp +++ b/mittest/simple_server/test_ls_recover.cpp @@ -256,6 +256,7 @@ TEST_F(ObLSBeforeRestartTest, create_unfinished_ls_without_disk) ASSERT_EQ(OB_SUCCESS, tenant_guard.switch_to(tenant_id)); ObCreateLSArg arg; + ObMajorMVMergeInfo major_mv_merge_info; ObLS *ls = NULL; ObLSService* ls_svr = MTL(ObLSService*); int64_t ls_epoch = 0; @@ -269,6 +270,7 @@ TEST_F(ObLSBeforeRestartTest, create_unfinished_ls_without_disk) migration_status, ObLSRestoreStatus(ObLSRestoreStatus::NONE), arg.get_create_scn(), + major_mv_merge_info, ObLSStoreFormat(ObLSStoreType::OB_LS_STORE_NORMAL), ls)); ObLSLockGuard lock_ls(ls); @@ -285,6 +287,7 @@ TEST_F(ObLSBeforeRestartTest, create_unfinished_ls_with_disk) ASSERT_EQ(OB_SUCCESS, tenant_guard.switch_to(tenant_id)); ObCreateLSArg arg; + ObMajorMVMergeInfo major_mv_merge_info; ObLS *ls = NULL; ObLSService* ls_svr = MTL(ObLSService*); ObLSID id_101(101); @@ -298,6 +301,7 @@ TEST_F(ObLSBeforeRestartTest, create_unfinished_ls_with_disk) migration_status, ObLSRestoreStatus(ObLSRestoreStatus::NONE), arg.get_create_scn(), + major_mv_merge_info, ObLSStoreFormat(ObLSStoreType::OB_LS_STORE_NORMAL), ls)); int64_t ls_epoch = 0; @@ -321,6 +325,7 @@ TEST_F(ObLSBeforeRestartTest, create_unfinished_ls_with_inner_tablet) ASSERT_EQ(OB_SUCCESS, tenant_guard.switch_to(tenant_id)); ObCreateLSArg arg; + ObMajorMVMergeInfo major_mv_merge_info; ObLS *ls = NULL; ObLSService* ls_svr = MTL(ObLSService*); ObLSID id_102(102); @@ -334,6 +339,7 @@ TEST_F(ObLSBeforeRestartTest, create_unfinished_ls_with_inner_tablet) migration_status, ObLSRestoreStatus(ObLSRestoreStatus::NONE), arg.get_create_scn(), + major_mv_merge_info, ObLSStoreFormat(ObLSStoreType::OB_LS_STORE_NORMAL), ls)); const bool unused_allow_log_sync = true; @@ -359,6 +365,7 @@ TEST_F(ObLSBeforeRestartTest, create_unfinished_ls_with_commit_slog) ASSERT_EQ(OB_SUCCESS, tenant_guard.switch_to(tenant_id)); ObCreateLSArg arg; + ObMajorMVMergeInfo major_mv_merge_info; ObLS *ls = NULL; ObLSService* ls_svr = MTL(ObLSService*); ObLSID id_103(103); @@ -372,6 +379,7 @@ TEST_F(ObLSBeforeRestartTest, create_unfinished_ls_with_commit_slog) migration_status, ObLSRestoreStatus(ObLSRestoreStatus::NONE), arg.get_create_scn(), + major_mv_merge_info, ObLSStoreFormat(ObLSStoreType::OB_LS_STORE_NORMAL), ls)); const bool unused_allow_log_sync = true; @@ -399,6 +407,7 @@ TEST_F(ObLSBeforeRestartTest, create_restore_ls) ASSERT_EQ(OB_SUCCESS, tenant_guard.switch_to(tenant_id)); ObCreateLSArg arg; + ObMajorMVMergeInfo major_mv_merge_info; ObLS *ls = NULL; ObLSService* ls_svr = MTL(ObLSService*); ObLSID id_104(104); @@ -413,6 +422,7 @@ TEST_F(ObLSBeforeRestartTest, create_restore_ls) migration_status, ObLSRestoreStatus(ObLSRestoreStatus::RESTORE_START), arg.get_create_scn(), + major_mv_merge_info, ObLSStoreFormat(ObLSStoreType::OB_LS_STORE_NORMAL), ls)); const bool unused_allow_log_sync = true; @@ -445,6 +455,7 @@ TEST_F(ObLSBeforeRestartTest, create_rebuild_ls) ASSERT_EQ(OB_SUCCESS, tenant_guard.switch_to(tenant_id)); ObCreateLSArg arg; + ObMajorMVMergeInfo major_mv_merge_info; ObLS *ls = NULL; ObLSService* ls_svr = MTL(ObLSService*); ObLSID id_105(105); @@ -459,6 +470,7 @@ TEST_F(ObLSBeforeRestartTest, create_rebuild_ls) migration_status, ObLSRestoreStatus(ObLSRestoreStatus::NONE), arg.get_create_scn(), + major_mv_merge_info, ObLSStoreFormat(ObLSStoreType::OB_LS_STORE_NORMAL), ls)); const bool unused_allow_log_sync = true; diff --git a/src/logservice/CMakeLists.txt b/src/logservice/CMakeLists.txt index d45fd1febf..3f60694d17 100644 --- a/src/logservice/CMakeLists.txt +++ b/src/logservice/CMakeLists.txt @@ -55,6 +55,7 @@ ob_set_subtarget(ob_logservice common ob_log_external_storage_utils.cpp ob_log_monitor.cpp ob_locality_adapter.cpp + ob_reconfig_checker_adapter.cpp ) ob_set_subtarget(ob_logservice common_mixed diff --git a/src/logservice/logrpc/ob_log_request_handler.cpp b/src/logservice/logrpc/ob_log_request_handler.cpp index 73232fd6f2..4365847453 100755 --- a/src/logservice/logrpc/ob_log_request_handler.cpp +++ b/src/logservice/logrpc/ob_log_request_handler.cpp @@ -15,6 +15,7 @@ #include "logservice/ob_log_handler.h" #include "logservice/logrpc/ob_log_rpc_proxy.h" #include "storage/tx_storage/ob_ls_handle.h" +#include "logservice/ob_reconfig_checker_adapter.h" #include "logservice/palf/log_define.h" #include "logservice/replayservice/ob_log_replay_service.h" @@ -232,12 +233,18 @@ int ConfigChangeCmdHandler::handle_config_change_cmd(const LogConfigChangeCmd &r { int ret = OB_SUCCESS; ObLogReporterAdapter *reporter; + logservice::ObReconfigCheckerAdapter reconfig_checker; if (NULL == palf_handle_) { ret = OB_NOT_INIT; } else if (false == req.is_valid()) { ret = OB_INVALID_ARGUMENT; } else if (OB_FAIL(get_reporter_(reporter))) { CLOG_LOG(ERROR, "get_reporter failed", K(req.palf_id_)); + } else if (OB_FAIL(reconfig_checker.init(MTL_ID(), share::ObLSID(req.palf_id_), req.timeout_us_))) { + CLOG_LOG(WARN, "ObReconfigCheckerAdapter init failed", K(ret), K(req.palf_id_)); + } else if (OB_FAIL(palf_handle_->set_reconfig_checker_cb(&reconfig_checker))) { + CLOG_LOG(WARN, "set_reconfig_checker_cb failed, another reconfiguration is running", K(ret), K(req.palf_id_)); + ret = OB_EAGAIN; } else { switch (req.cmd_type_) { case FORCE_SINGLE_MEMBER_CMD: @@ -294,6 +301,7 @@ int ConfigChangeCmdHandler::handle_config_change_cmd(const LogConfigChangeCmd &r default: break; } + palf_handle_->reset_reconfig_checker_cb(); } resp.ret_ = ret; if (OB_SUCC(ret) && OB_FAIL(reporter->report_replica_info(req.palf_id_))) { diff --git a/src/logservice/ob_log_handler.cpp b/src/logservice/ob_log_handler.cpp index 41d3419a75..59f5a183e4 100755 --- a/src/logservice/ob_log_handler.cpp +++ b/src/logservice/ob_log_handler.cpp @@ -578,6 +578,27 @@ int ObLogHandler::get_global_learner_list(common::GlobalLearnerList &learner_lis return palf_handle_.get_global_learner_list(learner_list); } +int ObLogHandler::get_stable_membership( + palf::LogConfigVersion &config_version, + common::ObMemberList &member_list, + int64_t &paxos_replica_num, + common::GlobalLearnerList &learner_list) const +{ + int ret = OB_SUCCESS; + RLockGuard guard(lock_); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + CLOG_LOG(WARN, "loghandler is not inited or maybe destroyed", K(ret), K(id_)); + } else if (is_in_stop_state_) { + ret = OB_NOT_RUNNING; + CLOG_LOG(INFO, "loghandler is stopped", K(ret), K_(id)); + } else if (OB_FAIL(palf_handle_.get_stable_membership(config_version, + member_list, paxos_replica_num, learner_list))) { + CLOG_LOG(WARN, "get_stable_membership failed", K(ret), KPC(this)); + } else {/*do nothing*/} + return ret; +} + int ObLogHandler::get_election_leader(common::ObAddr &addr) const { RLockGuard guard(lock_); diff --git a/src/logservice/ob_log_handler.h b/src/logservice/ob_log_handler.h index a4cca52a73..73751a5874 100755 --- a/src/logservice/ob_log_handler.h +++ b/src/logservice/ob_log_handler.h @@ -134,6 +134,10 @@ public: common::GlobalLearnerList &learner_list) const = 0; virtual int get_global_learner_list(common::GlobalLearnerList &learner_list) const = 0; virtual int get_leader_config_version(palf::LogConfigVersion &config_version) const = 0; + virtual int get_stable_membership(palf::LogConfigVersion &config_version, + common::ObMemberList &member_list, + int64_t &paxos_replica_num, + common::GlobalLearnerList &learner_list) const = 0; // get leader from election, used only for non_palf_leader rebuilding. virtual int get_election_leader(common::ObAddr &addr) const = 0; virtual int get_parent(common::ObAddr &parent) const = 0; @@ -481,6 +485,20 @@ public: // @brief, get global learner list of this paxos group // @param[out] common::GlobalLearnerList& int get_global_learner_list(common::GlobalLearnerList &learner_list) const override final; + // @brief, get stable membership info from the leader + // @param[out] palf::LogConfigVersion& + // @param[out] common::ObMemberList& + // @param[out] int64_t& + // @param[out] common::GlobalLearnerList& + // retval: + // OB_SUCCESS + // OB_NOT_INIT + // OB_EAGAIN: the leader is changing memberlist, try again + // OB_NOT_RUNNING: the replica is offlined + int get_stable_membership(palf::LogConfigVersion &config_version, + common::ObMemberList &member_list, + int64_t &paxos_replica_num, + common::GlobalLearnerList &learner_list) const override final; // @brief, get leader from election, used only for non_palf_leader rebuilding // @param[out] addr: address of leader // retval: diff --git a/src/logservice/ob_reconfig_checker_adapter.cpp b/src/logservice/ob_reconfig_checker_adapter.cpp new file mode 100644 index 0000000000..58b094e9dc --- /dev/null +++ b/src/logservice/ob_reconfig_checker_adapter.cpp @@ -0,0 +1,87 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ob_reconfig_checker_adapter.h" +#include "logservice/palf/log_define.h" +#include "rootserver/mview/ob_collect_mv_merge_info_task.h" +#include "share/ob_rpc_struct.h" +#include "storage/mview/ob_major_mv_merge_info.h" + +namespace oceanbase +{ +namespace logservice +{ + +ObReconfigCheckerAdapter::ObReconfigCheckerAdapter() +{ + tenant_id_ = OB_INVALID_TENANT_ID; + ls_id_.reset(); + timeout_ = -1; +} + +int ObReconfigCheckerAdapter::init(const uint64_t tenant_id, + const share::ObLSID &ls_id, + const int64_t &timeout) +{ + int ret = OB_SUCCESS; + + if (OB_INVALID_TENANT_ID == tenant_id + || !ls_id.is_valid() + || timeout < 0) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "invalid arguments", K(ret), K(tenant_id), K(ls_id), K(timeout)); + } else { + tenant_id_ = tenant_id; + ls_id_ = ls_id; + timeout_ = timeout; + } + + return ret; +} + +int ObReconfigCheckerAdapter::check_can_add_member(const ObAddr &server, + const int64_t timeout_us) +{ + int ret = OB_SUCCESS; + + const int64_t start_time = fast_current_time(); + const int64_t sleep_time = 300 * 1000; // 300ms + do { + if (OB_FAIL(ObMVCheckReplicaHelper::check_can_add_member(server, tenant_id_, ls_id_, timeout_us))) { + PALF_LOG(WARN, "check can add member failed", K(ret), + K(timeout_us), K(ls_id_), K(tenant_id_)); + } + if (fast_current_time() - start_time >= timeout_us) { + // if passed check, not return timeout + ret = OB_SUCC(ret) ? OB_SUCCESS : OB_TIMEOUT; + PALF_LOG(WARN, "check can add member timeout", K(ret), KPC(this), K(start_time), K(timeout_us)); + } + // if check failed, retry to timeout + if (OB_FAIL(ret) && ret != OB_TIMEOUT) { + usleep(sleep_time); + } + } while (OB_FAIL(ret) && ret != OB_TIMEOUT); + + return ret; +} + +int ObReconfigCheckerAdapter::check_can_change_memberlist(const ObMemberList &new_member_list, + const int64_t paxos_replica_num, + const int64_t timeout_us) +{ + int ret = OB_SUCCESS; + UNUSEDx(new_member_list, paxos_replica_num, timeout_us); + return ret; +} + +} // end namespace logservice +} // end namespace oceanbase diff --git a/src/logservice/ob_reconfig_checker_adapter.h b/src/logservice/ob_reconfig_checker_adapter.h new file mode 100644 index 0000000000..6a2e7f8ffa --- /dev/null +++ b/src/logservice/ob_reconfig_checker_adapter.h @@ -0,0 +1,47 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_LOGSERVICE_OB_RECONFIG_CHECKER_ADAPTER_H_ +#define OCEANBASE_LOGSERVICE_OB_RECONFIG_CHECKER_ADAPTER_H_ + +#include +#include "logservice/palf/palf_callback.h" +#include "lib/net/ob_addr.h" +#include "rootserver/ob_ls_recovery_stat_handler.h" + +namespace oceanbase +{ +namespace logservice +{ +class ObReconfigCheckerAdapter : public palf::PalfReconfigCheckerCb +{ +public: + explicit ObReconfigCheckerAdapter(); + virtual ~ObReconfigCheckerAdapter() { } + int init(const uint64_t tenant_id, const share::ObLSID &ls_id, const int64_t &timeout = -1); + TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(timeout)); +public: + virtual int check_can_add_member(const ObAddr &server, + const int64_t timeout_us) override final; + virtual int check_can_change_memberlist(const ObMemberList &new_member_list, + const int64_t paxos_replica_num, + const int64_t timeout_us) override final; +private: + uint64_t tenant_id_; + share::ObLSID ls_id_; + int64_t timeout_; +}; + +} // logservice +} // oceanbase + +#endif diff --git a/src/logservice/palf/palf_callback.h b/src/logservice/palf/palf_callback.h index 4ae8800f56..22add8fb63 100644 --- a/src/logservice/palf/palf_callback.h +++ b/src/logservice/palf/palf_callback.h @@ -142,6 +142,16 @@ public: virtual int get_server_region(const common::ObAddr &server, common::ObRegion ®ion) const = 0; }; +class PalfReconfigCheckerCb +{ +public: + virtual int check_can_add_member(const ObAddr &server, + const int64_t timeout_us) = 0; + virtual int check_can_change_memberlist(const ObMemberList &new_member_list, + const int64_t paxos_replica_num, + const int64_t timeout_us) = 0; +}; + } // end namespace palf } // end namespace oceanbase #endif diff --git a/src/logservice/palf/palf_callback_wrapper.cpp b/src/logservice/palf/palf_callback_wrapper.cpp index d35466f9c3..6fb83e0ab7 100644 --- a/src/logservice/palf/palf_callback_wrapper.cpp +++ b/src/logservice/palf/palf_callback_wrapper.cpp @@ -177,7 +177,9 @@ LogPlugins::LogPlugins() palflite_monitor_lock_(), palflite_monitor_(NULL), locality_cb_lock_(), - locality_cb_(NULL) { } + locality_cb_(NULL), + reconfig_checker_cb_lock_(), + reconfig_checker_cb_(NULL) { } LogPlugins::~LogPlugins() { @@ -202,6 +204,10 @@ void LogPlugins::destroy() common::RWLock::WLockGuard guard(locality_cb_lock_); locality_cb_ = NULL; } + { + common::RWLock::WLockGuard guard(reconfig_checker_cb_lock_); + reconfig_checker_cb_ = NULL; + } } template<> @@ -324,5 +330,35 @@ int LogPlugins::del_plugin(PalfLocalityInfoCb *plugin) return ret; } +template<> +int LogPlugins::add_plugin(PalfReconfigCheckerCb *plugin) +{ + int ret = OB_SUCCESS; + common::RWLock::WLockGuard guard(reconfig_checker_cb_lock_); + if (OB_ISNULL(plugin)) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "Palf plugin is NULL", KP(plugin)); + } else if (OB_NOT_NULL(reconfig_checker_cb_)) { + ret = OB_OP_NOT_ALLOW; + PALF_LOG(INFO, "Palf plugin is not NULL", KP(plugin), KP_(reconfig_checker_cb)); + } else { + reconfig_checker_cb_ = plugin; + PALF_LOG(INFO, "add_plugin success", KP(plugin)); + } + return ret; +} + +template<> +int LogPlugins::del_plugin(PalfReconfigCheckerCb *plugin) +{ + int ret = OB_SUCCESS; + common::RWLock::WLockGuard guard(reconfig_checker_cb_lock_); + if (OB_NOT_NULL(reconfig_checker_cb_)) { + PALF_LOG(INFO, "del_plugin success", KP_(reconfig_checker_cb)); + reconfig_checker_cb_ = NULL; + } + return ret; +} + }; // end namespace palf }; // end namespace oceanbase diff --git a/src/logservice/palf/palf_callback_wrapper.h b/src/logservice/palf/palf_callback_wrapper.h index 2804cd1272..372d764a44 100644 --- a/src/logservice/palf/palf_callback_wrapper.h +++ b/src/logservice/palf/palf_callback_wrapper.h @@ -135,7 +135,10 @@ public: PALF_PLUGINS_DELEGATE_PTR(palflite_monitor_, palflite_monitor_lock_, record_create_or_delete_event); PALF_PLUGINS_DELEGATE_PTR(locality_cb_, locality_cb_lock_, get_server_region); - TO_STRING_KV(KP_(loc_cb), KP_(palf_monitor), KP_(palflite_monitor)); + + PALF_PLUGINS_DELEGATE_PTR(reconfig_checker_cb_, reconfig_checker_cb_lock_, check_can_add_member); + PALF_PLUGINS_DELEGATE_PTR(reconfig_checker_cb_, reconfig_checker_cb_lock_, check_can_change_memberlist); + TO_STRING_KV(KP_(loc_cb), KP_(palf_monitor), KP_(palflite_monitor), KP_(reconfig_checker_cb)); private: common::RWLock loc_lock_; PalfLocationCacheCb *loc_cb_; @@ -145,6 +148,8 @@ private: PalfLiteMonitorCb *palflite_monitor_; common::RWLock locality_cb_lock_; PalfLocalityInfoCb *locality_cb_; + common::RWLock reconfig_checker_cb_lock_; + PalfReconfigCheckerCb *reconfig_checker_cb_; }; } // end namespace palf } // end namespace oceanbase diff --git a/src/logservice/palf/palf_handle.cpp b/src/logservice/palf/palf_handle.cpp index e1b2e07de4..b8955eba9c 100755 --- a/src/logservice/palf/palf_handle.cpp +++ b/src/logservice/palf/palf_handle.cpp @@ -322,6 +322,15 @@ int PalfHandle::get_paxos_member_list_and_learner_list(common::ObMemberList &mem return palf_handle_impl_->get_paxos_member_list_and_learner_list(member_list, paxos_replica_num, learner_list); } +int PalfHandle::get_stable_membership(LogConfigVersion &config_version, + common::ObMemberList &member_list, + int64_t &paxos_replica_num, + common::GlobalLearnerList &learner_list) const +{ + CHECK_VALID; + return palf_handle_impl_->get_stable_membership(config_version, member_list, paxos_replica_num, learner_list); +} + int PalfHandle::get_election_leader(common::ObAddr &addr) const { CHECK_VALID; @@ -753,6 +762,31 @@ int PalfHandle::reset_locality_cb() return ret; } +int PalfHandle::set_reconfig_checker_cb(palf::PalfReconfigCheckerCb *reconfig_checker) +{ + int ret = OB_SUCCESS; + CHECK_VALID; + if (OB_ISNULL(reconfig_checker)) { + PALF_LOG(INFO, "no need set_reconfig_checker_cb", KR(ret), KP(reconfig_checker)); + } else if (OB_FAIL(palf_handle_impl_->set_reconfig_checker_cb(reconfig_checker))) { + PALF_LOG(WARN, "set_reconfig_checker_cb failed", KR(ret)); + } else { + } + return ret; +} + +int PalfHandle::reset_reconfig_checker_cb() +{ + int ret = OB_SUCCESS; + CHECK_VALID; + if (OB_FAIL(palf_handle_impl_->reset_reconfig_checker_cb())) { + PALF_LOG(WARN, "reset_reconfig_checker_cb failed", KR(ret)); + } else { + PALF_LOG(INFO, "reset_reconfig_checker_cb success", KR(ret)); + } + return ret; +} + int PalfHandle::stat(PalfStat &palf_stat) const { CHECK_VALID; diff --git a/src/logservice/palf/palf_handle.h b/src/logservice/palf/palf_handle.h index 03a36d0540..6baf624d24 100755 --- a/src/logservice/palf/palf_handle.h +++ b/src/logservice/palf/palf_handle.h @@ -235,6 +235,10 @@ public: int get_paxos_member_list_and_learner_list(common::ObMemberList &member_list, int64_t &paxos_replica_num, GlobalLearnerList &learner_list) const; + int get_stable_membership(LogConfigVersion &config_version, + common::ObMemberList &member_list, + int64_t &paxos_replica_num, + common::GlobalLearnerList &learner_list) const; int get_election_leader(common::ObAddr &addr) const; int get_parent(common::ObAddr &parent) const; @@ -542,6 +546,8 @@ public: int reset_election_priority(); int set_locality_cb(palf::PalfLocalityInfoCb *locality_cb); int reset_locality_cb(); + int set_reconfig_checker_cb(palf::PalfReconfigCheckerCb *reconfig_checker); + int reset_reconfig_checker_cb(); int stat(PalfStat &palf_stat) const; //---------config change lock related--------// diff --git a/src/logservice/palf/palf_handle_impl.cpp b/src/logservice/palf/palf_handle_impl.cpp index 278c29a590..fe0150908b 100755 --- a/src/logservice/palf/palf_handle_impl.cpp +++ b/src/logservice/palf/palf_handle_impl.cpp @@ -592,6 +592,37 @@ int PalfHandleImpl::get_paxos_member_list_and_learner_list( return ret; } +int PalfHandleImpl::get_stable_membership(LogConfigVersion &config_version, + common::ObMemberList &member_list, + int64_t &paxos_replica_num, + common::GlobalLearnerList &learner_list) const +{ + int ret = OB_SUCCESS; + const int get_lock = config_change_lock_.trylock(); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + PALF_LOG(ERROR, "PalfHandleImpl has not inited", K(ret)); + } else if (OB_SUCCESS != get_lock) { + ret = OB_EAGAIN; + if (palf_reach_time_interval(1000 * 1000, config_change_print_time_us_)) { + PALF_LOG(WARN, "another config_change is running, try again", KR(ret), KPC(this)); + } + } else { + RLockGuard guard(lock_); + if (OB_FAIL(config_mgr_.get_config_version(config_version))) { + PALF_LOG(WARN, "failed to get_config_version", K(ret), K_(palf_id)); + } else if (OB_FAIL(config_mgr_.get_curr_member_list(member_list, paxos_replica_num))) { + PALF_LOG(WARN, "get_curr_member_list failed", K(ret), KPC(this)); + } else if (OB_FAIL(config_mgr_.get_global_learner_list(learner_list))) { + PALF_LOG(WARN, "get_global_learner_list failed", K(ret), KPC(this)); + } + } + if (OB_SUCCESS == get_lock) { + config_change_lock_.unlock(); + } + return ret; +} + int PalfHandleImpl::get_election_leader(ObAddr &addr) const { int ret = OB_SUCCESS; @@ -1406,7 +1437,20 @@ int PalfHandleImpl::one_stage_config_change_(const LogConfigChangeArgs &args, } } time_guard.click("precheck"); - // step 3: check whether the new config info can be set to the election module + // step 3: check whether the reconfiguration is allowed + if (OB_SUCC(ret) && is_add_log_sync_member_list(args.type_) && + UPGRADE_LEARNER_TO_ACCEPTOR != args.type_) { + ret = plugins_.check_can_add_member(args.server_.get_server(), timeout_us); + // reset retcode if the plugin is empty + ret = (OB_NOT_INIT == ret)? OB_SUCCESS: ret; + } else if (OB_SUCC(ret) && DEGRADE_ACCEPTOR_TO_LEARNER != args.type_ && + (is_remove_log_sync_member_list(args.type_) || CHANGE_REPLICA_NUM == args.type_)) { + ret = plugins_.check_can_change_memberlist(new_config_info.config_.log_sync_memberlist_, + new_config_info.config_.log_sync_replica_num_, timeout_us); + ret = (OB_NOT_INIT == ret)? OB_SUCCESS: ret; + } + time_guard.click("check_deps"); + // step 4: check whether the new config info can be set to the election module while (OB_SUCCESS == ret && OB_SUCC(not_timeout())) { { RLockGuard guard(lock_); @@ -1424,12 +1468,12 @@ int PalfHandleImpl::one_stage_config_change_(const LogConfigChangeArgs &args, ob_usleep(50 * 1000); } time_guard.click("wait_ele"); - // step 4: waiting for log barrier if a arbitration member exists + // step 5: waiting for log barrier if a arbitration member exists if (OB_SUCC(ret) && true == new_config_info.config_.arbitration_member_.is_valid()) { ret = wait_log_barrier_(args, new_config_info, not_timeout); } time_guard.click("wait_barrier"); - // step 5: motivate reconfiguration + // step 6: motivate reconfiguration while (OB_SUCCESS == ret && OB_SUCC(not_timeout())) { bool need_wlock = false; bool need_rlock = false; @@ -2592,6 +2636,35 @@ int PalfHandleImpl::reset_locality_cb() return ret; } +int PalfHandleImpl::set_reconfig_checker_cb(PalfReconfigCheckerCb *reconfig_checker) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + PALF_LOG(WARN, "not initted", KR(ret), KPC(this)); + } else if (OB_ISNULL(reconfig_checker)) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "reconfig_checker is NULL, can't register", KR(ret), KPC(this)); + } else if (OB_FAIL(plugins_.add_plugin(reconfig_checker))) { + PALF_LOG(WARN, "add_plugin failed", KR(ret), KPC(this), KP(reconfig_checker), K_(plugins)); + } else { + PALF_LOG(INFO, "set_reconfig_checker_cb success", KPC(this), K_(plugins), KP(reconfig_checker)); + } + return ret; +} + +int PalfHandleImpl::reset_reconfig_checker_cb() +{ + int ret = OB_SUCCESS; + PalfReconfigCheckerCb *reconfig_checker = NULL; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else if (OB_FAIL(plugins_.del_plugin(reconfig_checker))) { + PALF_LOG(WARN, "del_plugin failed", KR(ret), KPC(this), K_(plugins)); + } + return ret; +} + int PalfHandleImpl::check_and_switch_freeze_mode() { int ret = OB_SUCCESS; diff --git a/src/logservice/palf/palf_handle_impl.h b/src/logservice/palf/palf_handle_impl.h index 89d52d8435..a87b985b2d 100755 --- a/src/logservice/palf/palf_handle_impl.h +++ b/src/logservice/palf/palf_handle_impl.h @@ -315,6 +315,10 @@ public: virtual int get_paxos_member_list_and_learner_list(common::ObMemberList &member_list, int64_t &paxos_replica_num, common::GlobalLearnerList &learner_list) const = 0; + virtual int get_stable_membership(LogConfigVersion &config_version, + common::ObMemberList &member_list, + int64_t &paxos_replica_num, + common::GlobalLearnerList &learner_list) const = 0; virtual int get_election_leader(common::ObAddr &addr) const = 0; virtual int get_parent(common::ObAddr &parent) const = 0; @@ -760,6 +764,8 @@ public: virtual int reset_election_priority() = 0; virtual int set_locality_cb(palf::PalfLocalityInfoCb *locality_cb) = 0; virtual int reset_locality_cb() = 0; + virtual int set_reconfig_checker_cb(palf::PalfReconfigCheckerCb *reconfig_checker) = 0; + virtual int reset_reconfig_checker_cb() = 0; // ==================== Callback end ======================== virtual int advance_election_epoch_and_downgrade_priority(const int64_t proposal_id, const int64_t downgrade_priority_time_us, @@ -863,6 +869,10 @@ public: int get_paxos_member_list_and_learner_list(common::ObMemberList &member_list, int64_t &paxos_replica_num, common::GlobalLearnerList &learner_list) const override final; + int get_stable_membership(LogConfigVersion &config_version, + common::ObMemberList &member_list, + int64_t &paxos_replica_num, + common::GlobalLearnerList &learner_list) const override final; int get_election_leader(common::ObAddr &addr) const; int get_parent(common::ObAddr &parent) const; int force_set_as_single_replica() override final; @@ -971,6 +981,8 @@ public: int reset_election_priority() override final; int set_locality_cb(palf::PalfLocalityInfoCb *locality_cb) override final; int reset_locality_cb() override final; + int set_reconfig_checker_cb(palf::PalfReconfigCheckerCb *reconfig_checker) override final; + int reset_reconfig_checker_cb() override final; // ==================== Callback end ======================== public: int get_begin_lsn(LSN &lsn) const override final; @@ -1459,7 +1471,7 @@ private: // a spin lock for read/write replica_meta mutex SpinLock replica_meta_lock_; SpinLock rebuilding_lock_; - SpinLock config_change_lock_; + mutable SpinLock config_change_lock_; SpinLock mode_change_lock_; // a spin lock for single replica mutex SpinLock flashback_lock_; diff --git a/src/objit/include/objit/common/ob_item_type.h b/src/objit/include/objit/common/ob_item_type.h index 193418e02a..d02a743663 100644 --- a/src/objit/include/objit/common/ob_item_type.h +++ b/src/objit/include/objit/common/ob_item_type.h @@ -2599,8 +2599,12 @@ typedef enum ObItemType T_RB_ITERATE_EXPRESSION = 4737, T_MODULE_DATA = 4738, T_MODULE_NAME = 4739, + T_UNION_MERGE_HINT = 4740, T_UNION_MERGE_LIST = 4741, + + T_PSEUDO_OLD_NEW_COL = 4742, + T_MAX //Attention: add a new type before T_MAX } ObItemType; diff --git a/src/observer/ob_rpc_processor_simple.cpp b/src/observer/ob_rpc_processor_simple.cpp index 402948017b..2fb8fee8f1 100644 --- a/src/observer/ob_rpc_processor_simple.cpp +++ b/src/observer/ob_rpc_processor_simple.cpp @@ -90,6 +90,7 @@ #include "logservice/ob_server_log_block_mgr.h" #include "rootserver/ob_admin_drtask_util.h" #include "storage/ddl/ob_tablet_ddl_kv.h" +#include "storage/mview/ob_major_mv_merge_info.h" #ifdef OB_BUILD_SHARED_STORAGE #include "close_modules/shared_storage/storage/shared_storage/ob_ss_micro_cache.h" #include "close_modules/shared_storage/storage/shared_storage/ob_ss_micro_cache_io_helper.h" @@ -3583,6 +3584,97 @@ int ObResourceLimitCalculatorP::process() return ret; } +int ObCollectMvMergeInfoP::process() +{ + int ret = OB_SUCCESS; + ObMajorMVMergeInfo merge_info; + const share::ObLSID ls_id = arg_.get_ls_id(); + const uint64_t tenant_id = arg_.get_tenant_id(); + int64_t proposal_id = 0; + + MTL_SWITCH(tenant_id) { + if (arg_.need_update() && + OB_FAIL(ObMVCheckReplicaHelper::get_and_update_merge_info(ls_id, merge_info))) { + LOG_WARN("get and update merge info failed", K(ret)); + } else if (!arg_.need_update() && + OB_FAIL(ObMVCheckReplicaHelper::get_merge_info(ls_id, merge_info))) { + LOG_WARN("get merge info failed", K(ret)); + } else if (arg_.need_check_leader()) { + ObRole role; + logservice::ObLogService *log_service = nullptr; + if (OB_ISNULL(log_service = MTL(logservice::ObLogService*))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("log service should not be NULL", K(ret), KP(log_service)); + } else if (OB_FAIL(log_service->get_palf_role(ls_id, role, proposal_id))) { + LOG_WARN("failed to get role", K(ret), K(arg_)); + } else if (!is_strong_leader(role)) { + ret = OB_LS_NOT_LEADER; + LOG_WARN("it is not leader, cannot collect merge info", K(ret), K(ls_id), K(role), K(arg_)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(result_.init(merge_info, ret))) { + LOG_WARN("init collect mv merge info result failed", K(ret)); + } + } + return ret; +} + +int ObFetchStableMemberListP::process() +{ + int ret = OB_SUCCESS; + const share::ObLSID ls_id = arg_.get_ls_id(); + const uint64_t tenant_id = arg_.get_tenant_id(); + // todo siyu :: use new stable member list interface + MTL_SWITCH(tenant_id) { + ObLSService *ls_svr = NULL; + ObLSHandle ls_handle; + ObLS *ls = NULL; + logservice::ObLogHandler *log_handler = NULL; + common::ObMemberList member_list; + GlobalLearnerList learn_list; + int64_t paxos_replica_num = 0; + logservice::ObLogService *log_service = nullptr; + palf::LogConfigVersion log_config_version; + ObRole role; + int64_t proposal_id = 0; + int64_t proposal_id_new = 0; + + if (OB_ISNULL(log_service = MTL(logservice::ObLogService*))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("log service should not be NULL", K(ret), KP(log_service)); + } else if (OB_FAIL(log_service->get_palf_role(ls_id, role, proposal_id))) { + LOG_WARN("failed to get role", K(ret), K(arg_)); + } else if (!is_strong_leader(role)) { + ret = OB_LS_NOT_LEADER; + LOG_WARN("ls is not leader, cannot get member list", K(ret), K(role), K(arg_)); + } else if (OB_ISNULL(ls_svr = MTL(ObLSService *))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls service should not be null", K(ret)); + } else if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { + LOG_WARN("failed to get ls", K(ret), K(ls_id)); + } else if (OB_ISNULL(ls = ls_handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls should not be null", K(ret)); + } else if (OB_ISNULL(log_handler = ls->get_log_handler())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("log handler should not be NULL", K(ret)); + } else if (OB_FAIL(log_handler->get_stable_membership(log_config_version, member_list, + paxos_replica_num, learn_list))) { + LOG_WARN("failed to get paxos member list and log config version", K(ret)); + } else if (OB_FAIL(result_.init(member_list, log_config_version))) { + LOG_WARN("failed to int member list and config version", K(ret), K(member_list), K(log_config_version)); + } else if (OB_FAIL(log_service->get_palf_role(ls_id, role, proposal_id_new))) { + LOG_WARN("failed to get role", K(ret), K(arg_)); + } else if (proposal_id_new != proposal_id || !is_strong_leader(role)) { + // double check for get stable memberlist + ret = OB_LS_NOT_LEADER; + LOG_WARN("ls is not leader, cannot get member list", K(ret), K(role), K(arg_)); + } + } + return ret; +} + #ifdef OB_BUILD_SHARED_STORAGE int ObGetSSMacroBlockP::process() { diff --git a/src/observer/ob_rpc_processor_simple.h b/src/observer/ob_rpc_processor_simple.h index aca399ab18..0a38cca63f 100644 --- a/src/observer/ob_rpc_processor_simple.h +++ b/src/observer/ob_rpc_processor_simple.h @@ -307,6 +307,8 @@ OB_DEFINE_PROCESSOR_S(Srv, OB_CHECK_AND_CANCEL_DELETE_LOB_META_ROW_DAG, ObRpcChe OB_DEFINE_PROCESSOR_S(Srv, OB_KILL_CLIENT_SESSION, ObKillClientSessionP); OB_DEFINE_PROCESSOR_S(Srv, OB_CLIENT_SESSION_CONNECT_TIME, ObClientSessionConnectTimeP); +OB_DEFINE_PROCESSOR_S(Srv, OB_COLLECT_MV_MERGE_INFO, ObCollectMvMergeInfoP); +OB_DEFINE_PROCESSOR_S(Srv, OB_FETCH_STABLE_MEMBER_LIST, ObFetchStableMemberListP); OB_DEFINE_PROCESSOR_S(Srv, OB_CHANGE_EXTERNAL_STORAGE_DEST, ObRpcChangeExternalStorageDestP); OB_DEFINE_PROCESSOR_S(Srv, OB_KILL_QUERY_CLIENT_SESSION, ObKillQueryClientSessionP); OB_DEFINE_PROCESSOR_S(Srv, OB_NOTIFY_SHARED_STORAGE_INFO, ObNotifySharedStorageInfoP); diff --git a/src/observer/ob_srv_xlator_storage.cpp b/src/observer/ob_srv_xlator_storage.cpp index 37ac1b3f0d..45fdcb3784 100644 --- a/src/observer/ob_srv_xlator_storage.cpp +++ b/src/observer/ob_srv_xlator_storage.cpp @@ -145,6 +145,8 @@ void oceanbase::observer::init_srv_xlator_for_storage(ObSrvRpcXlator *xlator) { RPC_PROCESSOR(ObRpcTabletMajorFreezeP, gctx_); RPC_PROCESSOR(ObRpcDetectSessionAliveP, gctx_); RPC_PROCESSOR(ObCancelGatherStatsP, gctx_); + RPC_PROCESSOR(ObCollectMvMergeInfoP, gctx_); + RPC_PROCESSOR(ObFetchStableMemberListP, gctx_); RPC_PROCESSOR(ObRpcPrepareTabletSplitTaskRangesP, gctx_); RPC_PROCESSOR(ObRpcCheckStorageOperationStatusP, gctx_); #ifdef OB_BUILD_SHARED_STORAGE diff --git a/src/observer/table_load/ob_table_load_parallel_merge_ctx.cpp b/src/observer/table_load/ob_table_load_parallel_merge_ctx.cpp index 0493095259..38110303a3 100644 --- a/src/observer/table_load/ob_table_load_parallel_merge_ctx.cpp +++ b/src/observer/table_load/ob_table_load_parallel_merge_ctx.cpp @@ -397,7 +397,7 @@ public: } else { ObDirectLoadMultipleSSTableBuildParam build_param; build_param.tablet_id_ = tablet_ctx_->tablet_id_; - build_param.table_data_desc_ = parallel_merge_ctx_->table_data_desc_; + build_param.table_data_desc_ = parallel_merge_ctx_->store_ctx_->table_data_desc_; build_param.datum_utils_ = &(ctx_->schema_.datum_utils_); build_param.file_mgr_ = parallel_merge_ctx_->store_ctx_->tmp_file_mgr_; build_param.extra_buf_ = extra_buf_; @@ -456,7 +456,7 @@ public: int ret = OB_SUCCESS; ObDirectLoadMultipleSSTableCompactParam compact_param; compact_param.tablet_id_ = tablet_ctx_->tablet_id_; - compact_param.table_data_desc_ = parallel_merge_ctx_->table_data_desc_; + compact_param.table_data_desc_ = parallel_merge_ctx_->store_ctx_->table_data_desc_; compact_param.datum_utils_ = &ctx_->schema_.datum_utils_; if (OB_FAIL(compactor_.init(compact_param))) { LOG_WARN("fail to init sstable compactor", KR(ret)); diff --git a/src/observer/table_load/ob_table_load_service.cpp b/src/observer/table_load/ob_table_load_service.cpp index c8b1fe1688..5094939630 100644 --- a/src/observer/table_load/ob_table_load_service.cpp +++ b/src/observer/table_load/ob_table_load_service.cpp @@ -594,10 +594,10 @@ int ObTableLoadService::check_support_direct_load(ObSchemaGetterGuard &schema_gu FORWARD_USER_ERROR_MSG(ret, "%sdirect-load does not support table has roaringbitmap column", tmp_prefix); } // check if table has mlog - else if (table_schema->has_mlog_table()) { + else if (table_schema->required_by_mview_refresh()) { ret = OB_NOT_SUPPORTED; - LOG_WARN("direct-load does not support table with materialized view log", KR(ret)); - FORWARD_USER_ERROR_MSG(ret, "%sdirect-load does not support table with materialized view log", tmp_prefix); + LOG_WARN("direct-load does not support table required by materialized view refresh", KR(ret)); + FORWARD_USER_ERROR_MSG(ret, "%sdirect-load does not support table required by materialized view refresh", tmp_prefix); } else if (ObDirectLoadMethod::is_incremental(method)) { // incremental direct-load if (!ObDirectLoadInsertMode::is_valid_for_incremental_method(insert_mode)) { ret = OB_NOT_SUPPORTED; diff --git a/src/observer/virtual_table/ob_all_virtual_ls_info.cpp b/src/observer/virtual_table/ob_all_virtual_ls_info.cpp index a8c43f5139..4b75343e8f 100644 --- a/src/observer/virtual_table/ob_all_virtual_ls_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_ls_info.cpp @@ -196,6 +196,18 @@ int ObAllVirtualLSInfo::process_curr_tenant(ObNewRow *&row) // required_data_disk_size cur_row_.cells_[i].set_int(ls_info.required_data_disk_size_); break; + case OB_APP_MIN_COLUMN_ID + 17: + // mv_major_merge_scn + cur_row_.cells_[i].set_uint64(!ls_info.mv_major_merge_scn_.is_valid() ? 0 : ls_info.mv_major_merge_scn_.get_val_for_inner_table_field()); + break; + case OB_APP_MIN_COLUMN_ID + 18: + // mv_publish_scn + cur_row_.cells_[i].set_uint64(!ls_info.mv_publish_scn_.is_valid() ? 0 : ls_info.mv_publish_scn_.get_val_for_inner_table_field()); + break; + case OB_APP_MIN_COLUMN_ID + 19: + // mv_publish_scn + cur_row_.cells_[i].set_uint64(!ls_info.mv_safe_scn_.is_valid() ? 0 : ls_info.mv_safe_scn_.get_val_for_inner_table_field()); + break; default: ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "invalid col_id", K(ret), K(col_id)); diff --git a/src/observer/virtual_table/ob_all_virtual_proxy_schema.cpp b/src/observer/virtual_table/ob_all_virtual_proxy_schema.cpp index 91f4340051..d01f94b1bf 100644 --- a/src/observer/virtual_table/ob_all_virtual_proxy_schema.cpp +++ b/src/observer/virtual_table/ob_all_virtual_proxy_schema.cpp @@ -858,8 +858,7 @@ int ObAllVirtualProxySchema::get_next_tablet_location_( } } if (OB_SUCC(ret)) { // dump each location - DupReplicaType dup_replica_type = ObDuplicateScope::DUPLICATE_SCOPE_CLUSTER - == table_schema->get_duplicate_scope() ? + DupReplicaType dup_replica_type = table_schema->is_duplicate_table() ? DupReplicaType::DUP_REPLICA : DupReplicaType::NON_DUP_REPLICA; const uint64_t table_id = table_schema->get_table_id(); diff --git a/src/observer/virtual_table/ob_information_kvcache_table.cpp b/src/observer/virtual_table/ob_information_kvcache_table.cpp index 50717148a6..1a08ddbd27 100644 --- a/src/observer/virtual_table/ob_information_kvcache_table.cpp +++ b/src/observer/virtual_table/ob_information_kvcache_table.cpp @@ -199,6 +199,8 @@ int ObInfoSchemaKvCacheTable::set_diagnose_info(ObKVCacheInst *inst, ObDiagnoseT inst->status_.total_miss_cnt_ = GLOBAL_EVENT_GET(ObStatEventIds::OPT_DS_STAT_CACHE_MISS); } else if (0 == strcmp(inst->status_.config_->cache_name_,"log_kv_cache")) { inst->status_.total_miss_cnt_ = GLOBAL_EVENT_GET(ObStatEventIds::LOG_KV_CACHE_MISS); + } else if (0 == strcmp(inst->status_.config_->cache_name_,"multi_version_fuse_row_cache")) { + inst->status_.total_miss_cnt_ = GLOBAL_EVENT_GET(ObStatEventIds::MULTI_VERSION_FUSE_ROW_CACHE_MISS); } else if (0 == strcmp(inst->status_.config_->cache_name_,"tablet_table_cache")) { inst->status_.total_miss_cnt_ = GLOBAL_EVENT_GET(ObStatEventIds::TABLET_CACHE_MISS); } else if (0 == strcmp(inst->status_.config_->cache_name_,"storage_meta_cache")) { diff --git a/src/rootserver/CMakeLists.txt b/src/rootserver/CMakeLists.txt index 9ce81d214c..6e7577f02d 100644 --- a/src/rootserver/CMakeLists.txt +++ b/src/rootserver/CMakeLists.txt @@ -199,6 +199,12 @@ ob_set_subtarget(ob_rootserver mview mview/ob_mview_refresh_stats_maintenance_task.cpp mview/ob_mview_timer_task.cpp mview/ob_mview_dependency_service.cpp + mview/ob_mview_push_refresh_scn_task.cpp + mview/ob_mview_push_snapshot_task.cpp + mview/ob_replica_safe_check_task.cpp + mview/ob_collect_mv_merge_info_task.cpp + mview/ob_mview_clean_snapshot_task.cpp + mview/ob_mview_update_cache_task.cpp ) ob_set_subtarget(ob_rootserver direct_load diff --git a/src/rootserver/backup/ob_backup_data_scheduler.cpp b/src/rootserver/backup/ob_backup_data_scheduler.cpp index 4dbdf8e207..cb34f8ce51 100644 --- a/src/rootserver/backup/ob_backup_data_scheduler.cpp +++ b/src/rootserver/backup/ob_backup_data_scheduler.cpp @@ -27,6 +27,8 @@ #include "share/backup/ob_backup_connectivity.h" #include "rootserver/ob_rs_event_history_table_operator.h" #include "rootserver/restore/ob_restore_util.h" +#include "share/ob_global_stat_proxy.h" +#include "share/schema/ob_mview_info.h" namespace oceanbase { @@ -1152,7 +1154,9 @@ int ObUserTenantBackupJobMgr::process() ObBackupStatus::Status status = job_attr_->status_.status_; switch (status) { case ObBackupStatus::Status::INIT: { - if (OB_FAIL(check_dest_validity_())) { + if (OB_FAIL(select_mview_for_update_(gen_user_tenant_id(tenant_id_)))) { + LOG_WARN("failed to select mview for update", K(ret), K_(tenant_id)); + } else if (OB_FAIL(check_dest_validity_())) { LOG_WARN("[DATA_BACKUP]fail to check dest validity", K(ret)); } else if (OB_FAIL(persist_set_task_())) { LOG_WARN("[DATA_BACKUP]failed to persist log stream task", K(ret), KPC(job_attr_)); @@ -1605,6 +1609,57 @@ int ObUserTenantBackupJobMgr::advance_job_status( return ret; } +int ObUserTenantBackupJobMgr::select_mview_for_update_(const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + ObMySQLTransaction trans; + ObAllTenantInfo tenant_info; + if (OB_INVALID_ID == tenant_id) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid args", K(ret), K(tenant_id)); + } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id, sql_proxy_, false/*for update*/, tenant_info))) { + LOG_WARN("failed to get tenant info", K(ret), K(tenant_id)); + } else if (!tenant_info.is_primary()) { + LOG_INFO("tenant is not primary", K(tenant_id), K(tenant_info)); + } else if (OB_FAIL(trans.start(sql_proxy_, tenant_id))) { + LOG_WARN("failed to start trans", K(ret), K(tenant_id)); + } else { + share::ObGlobalStatProxy stat_proxy(trans, tenant_id); + share::SCN major_refresh_mv_merge_scn; + const bool select_for_update = true; + bool mview_in_creation = false; + if (OB_FAIL(stat_proxy.get_major_refresh_mv_merge_scn(select_for_update, + major_refresh_mv_merge_scn))) { + if (OB_ERR_NULL_VALUE == ret) { + LOG_INFO("major_refresh_mv_merge_scn has not been set", K(tenant_id)); + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to get major_refresh_mv_merge_scn", K(ret), K(tenant_id)); + } + } else if (OB_UNLIKELY(!major_refresh_mv_merge_scn.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("major_refresh_mv_merge_scn is invalid", K(ret), K(tenant_id), K(major_refresh_mv_merge_scn)); + } else if (OB_FAIL(ObMViewInfo::contains_major_refresh_mview_in_creation(trans, tenant_id, mview_in_creation))) { + LOG_WARN("fail to check if mv is in creation", K(ret), K(tenant_id)); + } else if (mview_in_creation) { + ret = OB_EAGAIN; + // when a mview is being created, its snapshot may be added but the dependency is not + // added yet, which can cause cleaning the snapshot by mistake. so we just skip this round. + LOG_INFO("mview is being created, skip clean task", K(ret), K(tenant_id)); + } + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(trans.end(OB_SUCC(ret)))) { + LOG_WARN("failed to commit trans", K(ret), K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } else { + LOG_INFO("no mview is being created", K(ret), K(tenant_id)); + } + } + } + return ret; +} + /* *-----------------------ObSysTenantBackupJobMgr------------------------ */ diff --git a/src/rootserver/backup/ob_backup_data_scheduler.h b/src/rootserver/backup/ob_backup_data_scheduler.h index 12089f6884..6b00e10319 100644 --- a/src/rootserver/backup/ob_backup_data_scheduler.h +++ b/src/rootserver/backup/ob_backup_data_scheduler.h @@ -162,6 +162,7 @@ private: int check_dest_validity_(); int cancel_(); int update_set_task_to_canceling_(); + int select_mview_for_update_(const uint64_t tenant_id); private: DISALLOW_COPY_AND_ASSIGN(ObUserTenantBackupJobMgr); }; diff --git a/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp b/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp index 81ef136b30..779b5f823b 100644 --- a/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp +++ b/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp @@ -589,6 +589,8 @@ int ObBackupSetTaskMgr::backup_meta_finish_() LOG_WARN("fail to merge ls meta infos", K(ret), K(ls_task)); } else if (OB_FAIL(merge_tablet_to_ls_info_(consistent_scn, ls_task, new_ls_ids))) { LOG_WARN("[DATA_BACKUP]failed to merge tablet to ls info", K(ret), K(ls_task)); + } else if (OB_FAIL(backup_major_compaction_mview_dep_tablet_list_())) { + LOG_WARN("failed to backup mview dep tablet list", K(ret)); } else if (OB_FALSE_IT(DEBUG_SYNC(BEFORE_BACKUP_DATA))) { } else if (OB_FAIL(trans_.start(sql_proxy_, meta_tenant_id_))) { LOG_WARN("fail to start trans", K(ret), K(meta_tenant_id_)); @@ -821,6 +823,28 @@ int ObBackupSetTaskMgr::merge_tablet_to_ls_info_(const share::SCN &consistent_sc return ret; } +int ObBackupSetTaskMgr::backup_major_compaction_mview_dep_tablet_list_() +{ + int ret = OB_SUCCESS; + ObBackupMajorCompactionMViewDepTabletListDesc desc; + common::ObArray mview_tablet_list; + share::SCN backup_scn; + if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy should not be null", K(ret)); + } else if (OB_FAIL(ObBackupDataScheduler::get_backup_scn(*sql_proxy_, set_task_attr_.tenant_id_, true/*is_backup_start*/, backup_scn))) { + LOG_WARN("failed to get backup scn", K(ret), K(set_task_attr_)); + } else if (OB_FAIL(ObBackupMViewOperator::get_all_major_compaction_mview_dep_tablet_list( + *sql_proxy_, set_task_attr_.tenant_id_, backup_scn, mview_tablet_list))) { + LOG_WARN("failed to get all major compaction mveiw dep tablet list", K(ret), K(backup_scn)); + } else if (OB_FAIL(desc.tablet_id_list_.assign(mview_tablet_list))) { + LOG_WARN("failed to assign tablet list", K(ret)); + } else if (OB_FAIL(store_.write_major_compaction_mview_dep_tablet_list(desc))) { + LOG_WARN("failed to write mview dep tablet list", K(ret)); + } + return ret; +} + int ObBackupSetTaskMgr::get_tablet_list_by_snapshot( const share::SCN &consistent_scn, common::hash::ObHashMap> &latest_ls_tablet_map) { diff --git a/src/rootserver/backup/ob_backup_data_set_task_mgr.h b/src/rootserver/backup/ob_backup_data_set_task_mgr.h index 4829538e43..82a7a6eae6 100644 --- a/src/rootserver/backup/ob_backup_data_set_task_mgr.h +++ b/src/rootserver/backup/ob_backup_data_set_task_mgr.h @@ -66,6 +66,7 @@ private: int merge_tablet_to_ls_info_(const share::SCN &consistent_scn, const ObIArray &ls_tasks, common::ObIArray &ls_ids); + int backup_major_compaction_mview_dep_tablet_list_(); int get_tablet_list_by_snapshot( const share::SCN &consistent_scn, common::hash::ObHashMap> &latest_ls_tablet_map); int fill_map_with_sys_tablets_(common::hash::ObHashMap> &latest_ls_tablet_map); diff --git a/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp b/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp index 05a3eb03cc..a4c08b32cc 100644 --- a/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp +++ b/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp @@ -345,6 +345,134 @@ int ObDDLRedefinitionTask::check_table_empty(const ObDDLTaskStatus next_task_sta return ret; } +int ObDDLRedefinitionTask::add_table_tablets_for_snapshot_( + const uint64_t table_id, ObSchemaGetterGuard &schema_guard, + common::ObIArray &tablet_ids) +{ + int ret = OB_SUCCESS; + const ObTableSchema *table_schema = NULL; + + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("ObDDLRedefinitionTask has not been inited", K(ret)); + } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id_, table_id, table_schema))) { + LOG_WARN("get table schema failed", K(ret), K(object_id_)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("table not exist", K(ret), K(table_id)); + } else if (OB_FAIL(ObDDLUtil::get_tablets(tenant_id_, table_id, tablet_ids))) { + LOG_WARN("failed to get data table snapshot", K(ret)); + } else if (table_schema->get_aux_lob_meta_tid() != OB_INVALID_ID && + OB_FAIL(ObDDLUtil::get_tablets(tenant_id_, table_schema->get_aux_lob_meta_tid(), + tablet_ids))) { + LOG_WARN("failed to get data lob meta table snapshot", K(ret)); + } else if (table_schema->get_aux_lob_piece_tid() != OB_INVALID_ID && + OB_FAIL(ObDDLUtil::get_tablets(tenant_id_, table_schema->get_aux_lob_piece_tid(), + tablet_ids))) { + LOG_WARN("failed to get data lob piece table snapshot", K(ret)); + } + + return ret; +} + +int ObDDLRedefinitionTask::hold_snapshot_for_major_refresh_mv_(const int64_t snapshot_version) +{ + int ret = OB_SUCCESS; + ObSEArray tablet_ids; + ObRootService *root_service = GCTX.root_service_; + ObMultiVersionSchemaService &schema_service = ObMultiVersionSchemaService::get_instance(); + ObSchemaGetterGuard schema_guard; + SCN snapshot_scn; + + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("ObDDLRedefinitionTask has not been inited", K(ret)); + } else if (OB_ISNULL(root_service)) { + ret = OB_ERR_SYS; + LOG_WARN("error sys, root service must not be nullptr", K(ret)); + } else if (OB_FAIL(snapshot_scn.convert_for_tx(snapshot_version))) { + LOG_WARN("failed to convert", K(snapshot_version), K(ret)); + } else if (OB_FAIL(schema_service.get_tenant_schema_guard(tenant_id_, schema_guard))) { + LOG_WARN("get tenant schema guard failed", K(ret)); + } else if (OB_FAIL(add_table_tablets_for_snapshot_(target_object_id_, schema_guard, tablet_ids))) { + // target_table and src_table will be exchanged after loading data, which means target_table + // will become the container table of MV at last, so we should hold its snapshot. + LOG_WARN("failed to add table tablets for snapshot", K(ret), K(object_id_), + K(target_object_id_)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < alter_table_arg_.based_schema_object_infos_.count(); + ++i) { + const ObBasedSchemaObjectInfo &base_info = alter_table_arg_.based_schema_object_infos_.at(i); + if (ObSchemaType::TABLE_SCHEMA == base_info.schema_type_ && + OB_FAIL(add_table_tablets_for_snapshot_(base_info.schema_id_, schema_guard, tablet_ids))) { + LOG_WARN("failed to get base table snapshot", K(ret), K(base_info.schema_id_)); + } + } + if (OB_SUCC(ret)) { + ObDDLService &ddl_service = root_service->get_ddl_service(); + if (OB_FAIL(ddl_service.get_snapshot_mgr().batch_acquire_snapshot( + ddl_service.get_sql_proxy(), SNAPSHOT_FOR_MAJOR_REFRESH_MV, tenant_id_, + schema_version_, snapshot_scn, nullptr, tablet_ids))) { + LOG_WARN("batch acquire snapshot failed", K(ret), K(tablet_ids)); + } + LOG_INFO("[MAJ_REF_MV] hold snapshot for major refresh mv", KR(ret), K(snapshot_scn), + K(tablet_ids)); + } + } + + return ret; +} + +int ObDDLRedefinitionTask::release_snapshot(const int64_t snapshot_version) +{ + int ret = OB_SUCCESS; + ObRootService *root_service = GCTX.root_service_; + ObSEArray tablet_ids; + ObSchemaGetterGuard schema_guard; + const ObTableSchema *data_table_schema = nullptr; + const ObTableSchema *dest_table_schema = nullptr; + ObMultiVersionSchemaService &schema_service = ObMultiVersionSchemaService::get_instance(); + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_ISNULL(root_service)) { + ret = OB_ERR_SYS; + LOG_WARN("error sys, root service must not be nullptr", K(ret)); + } else if (OB_FAIL(DDL_SIM(tenant_id_, task_id_, DDL_TASK_RELEASE_SNAPSHOT_FAILED))) { + LOG_WARN("ddl sim failure", K(ret), K(tenant_id_), K(task_id_)); + } else if (OB_FAIL(schema_service.get_tenant_schema_guard(tenant_id_, schema_guard))) { + LOG_WARN("get tenant schema guard failed", K(ret)); + } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id_, object_id_, data_table_schema))) { + LOG_WARN("get table schema failed", K(ret), K(object_id_)); + } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id_, target_object_id_, dest_table_schema))) { + LOG_WARN("get table schema failed", K(ret), K(target_object_id_)); + } else if (OB_ISNULL(data_table_schema) || OB_ISNULL(dest_table_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("table not exist", K(ret), K(object_id_), K(target_object_id_), KP(data_table_schema), KP(dest_table_schema)); + } else if (OB_FAIL(ObDDLUtil::get_tablets(tenant_id_, object_id_, tablet_ids))) { + LOG_WARN("failed to get data table snapshot", K(ret)); + } else if (OB_FAIL(ObDDLUtil::get_tablets(tenant_id_, target_object_id_, tablet_ids))) { + LOG_WARN("failed to get dest table snapshot", K(ret)); + } else if (data_table_schema->get_aux_lob_meta_tid() != OB_INVALID_ID && + OB_FAIL(ObDDLUtil::get_tablets(tenant_id_, data_table_schema->get_aux_lob_meta_tid(), tablet_ids))) { + LOG_WARN("failed to get data lob meta table snapshot", K(ret)); + } else if (data_table_schema->get_aux_lob_piece_tid() != OB_INVALID_ID && + OB_FAIL(ObDDLUtil::get_tablets(tenant_id_, data_table_schema->get_aux_lob_piece_tid(), tablet_ids))) { + LOG_WARN("failed to get data lob piece table snapshot", K(ret)); + } else if (dest_table_schema->get_aux_lob_meta_tid() != OB_INVALID_ID && + OB_FAIL(ObDDLUtil::get_tablets(tenant_id_, dest_table_schema->get_aux_lob_meta_tid(), tablet_ids))) { + LOG_WARN("failed to get dest lob meta table snapshot", K(ret)); + } else if (dest_table_schema->get_aux_lob_piece_tid() != OB_INVALID_ID && + OB_FAIL(ObDDLUtil::get_tablets(tenant_id_, dest_table_schema->get_aux_lob_piece_tid(), tablet_ids))) { + LOG_WARN("failed to get dest lob piece table snapshot", K(ret)); + } else if (OB_FAIL(batch_release_snapshot(snapshot_version, tablet_ids))) { + LOG_WARN("failed to release snapshot", K(ret)); + } + add_event_info("release snapshot finish"); + LOG_INFO("release snapshot finished", K(ret), K(snapshot_version), K(object_id_), K(target_object_id_), K(schema_version_), "ddl_event_info", ObDDLEventInfo()); + return ret; +} + // to hold snapshot, containing data in old table with new schema version. int ObDDLRedefinitionTask::obtain_snapshot(const ObDDLTaskStatus next_task_status) { @@ -358,6 +486,10 @@ int ObDDLRedefinitionTask::obtain_snapshot(const ObDDLTaskStatus next_task_statu target_object_id_, snapshot_version_, snapshot_held_, this))) { LOG_WARN("fail to obtain_snapshot", K(ret), K(snapshot_version_), K(snapshot_held_)); + } else if (alter_table_arg_.mview_refresh_info_.is_mview_complete_refresh_ && + alter_table_arg_.alter_table_schema_.mv_major_refresh() && + OB_FAIL(hold_snapshot_for_major_refresh_mv_(snapshot_version_))) { + LOG_WARN("failed to hold snapshot for major refresh mv", K(ret), K(snapshot_version_)); } return ret; } diff --git a/src/rootserver/ddl_task/ob_ddl_redefinition_task.h b/src/rootserver/ddl_task/ob_ddl_redefinition_task.h index 4cd669b075..2a638b1a64 100644 --- a/src/rootserver/ddl_task/ob_ddl_redefinition_task.h +++ b/src/rootserver/ddl_task/ob_ddl_redefinition_task.h @@ -174,6 +174,8 @@ protected: virtual int check_and_cancel_complement_data_dag(bool &all_complement_dag_exit); // wait dag exit before unlock table. virtual int fail(); virtual int success(); + int hold_snapshot(const int64_t snapshot_version); + int release_snapshot(const int64_t snapshot_version); int add_constraint_ddl_task(const int64_t constraint_id); int add_fk_ddl_task(const int64_t fk_id); int sync_auto_increment_position(); @@ -265,6 +267,10 @@ protected: obrpc::ObAlterTableArg &alter_table_arg); int64_t get_build_replica_request_time(); private: + int add_table_tablets_for_snapshot_(const uint64_t table_id, ObSchemaGetterGuard &schema_guard, + common::ObIArray &tablet_ids); + int hold_snapshot_for_major_refresh_mv_(const int64_t snapshot_version); + virtual int cleanup_impl() override; protected: static const int64_t MAP_BUCKET_NUM = 1024; diff --git a/src/rootserver/mview/ob_collect_mv_merge_info_task.cpp b/src/rootserver/mview/ob_collect_mv_merge_info_task.cpp new file mode 100644 index 0000000000..96e3d5d888 --- /dev/null +++ b/src/rootserver/mview/ob_collect_mv_merge_info_task.cpp @@ -0,0 +1,509 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX RS +#include "rootserver/mview/ob_collect_mv_merge_info_task.h" +#include "lib/ob_define.h" +#include "observer/ob_server.h" +#include "observer/ob_server_struct.h" +#include "src/storage/high_availability/ob_storage_ha_src_provider.h" +#include "share/ob_global_stat_proxy.h" + +namespace oceanbase +{ +namespace rootserver +{ +int ObCollectMvMergeInfoTask::init() +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObCollectMvMergeInfoTask init twice", KR(ret), KP(this)); + } else { + const uint64_t tenant_id = MTL_ID(); + tenant_id_ = tenant_id; + is_inited_ = true; + } + return ret; +} + +int ObCollectMvMergeInfoTask::start() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObMViewMaintenanceTask not init", KR(ret), KP(this)); + } else { + is_stop_ = false; + if (!in_sched_ && OB_FAIL(schedule_task(SLEEP_SECONDS, true /*repeat*/))) { + LOG_WARN("fail to schedule mview maintenance task", KR(ret)); + } else { + in_sched_ = true; + } + } + return ret; +} + +void ObCollectMvMergeInfoTask::stop() +{ + is_stop_ = true; + in_sched_ = false; + cancel_task(); +} + +void ObCollectMvMergeInfoTask::wait() { wait_task(); } + +void ObCollectMvMergeInfoTask::destroy() +{ + is_inited_ = false; + is_stop_ = true; + in_sched_ = false; + cancel_task(); + wait_task(); + tenant_id_ = OB_INVALID_TENANT_ID; +} + +// todo siyu :: use new interface to get stable memberlist +int ObCollectMvMergeInfoTask::get_stable_member_list_and_config_version(const uint64_t tenant_id, + const share::ObLSID &ls_id, + common::ObIArray &addr_list, + palf::LogConfigVersion &log_config_version) +{ + int ret = OB_SUCCESS; + + using namespace storage; + ObLSService *ls_svr = NULL; + ObStorageRpc *storage_rpc = NULL; + ObStorageHAGetMemberHelper get_member_helper; + common::ObAddr leader_addr; + obrpc::ObFetchStableMemberListArg arg; + obrpc::ObFetchStableMemberListInfo member_info; + const uint64_t cluster_id = GCONF.cluster_id; + int64_t rpc_timeout = obrpc::ObRpcProxy::MAX_RPC_TIMEOUT; + + if (!ls_id.is_valid() || !is_valid_tenant_id(tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(ls_id), K(tenant_id)); + } else if (OB_ISNULL(ls_svr = (MTL(ObLSService *)))) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("ls service should not be NULL", KR(ret), KP(ls_svr)); + } else if (OB_ISNULL(storage_rpc = ls_svr->get_storage_rpc())) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("storage rpc shoulde not be null", KR(ret), KP(storage_rpc)); + } else if (OB_FAIL(arg.init(ls_id, tenant_id))) { + LOG_WARN("failed to fetch stable member list arg", KR(ret), K(arg)); + } else if (OB_FAIL(get_member_helper.init(storage_rpc))) { + LOG_WARN("failed to init get member helper", KR(ret), KP(storage_rpc)); + } else if (OB_FAIL(get_member_helper.get_ls_leader(tenant_id, ls_id, leader_addr))) { + LOG_WARN("fail to get ls leader", KR(ret), K(tenant_id), K(ls_id)); + } else if (OB_FAIL(GCTX.srv_rpc_proxy_->to(leader_addr) + .dst_cluster_id(cluster_id) + .by(tenant_id) + .timeout(rpc_timeout) + .fetch_stable_member_list(arg, member_info))) { + LOG_WARN("get ls member list failed", KR(ret), K(leader_addr), K(tenant_id)); + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(member_info.get_member_list().get_addr_array(addr_list))) { + LOG_WARN("failed to get member_list addr array", KR(ret), K(member_info)); + } + + return ret; +} + +int ObCollectMvMergeInfoTask::check_ls_attr_state_(const ObLSAttr &ls_attr) +{ + int ret = OB_SUCCESS; + if (!ls_attr.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(ls_attr)); + } else if (!ls_attr.ls_is_normal()) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("ls not in normal state, skip task", KR(ret), K(ls_attr)); + } + return ret; +} + +int ObCollectMvMergeInfoTask::check_ls_list_state(const ObLSAttrArray &ls_attr_array) +{ + int ret = OB_SUCCESS; + + ARRAY_FOREACH(ls_attr_array, i) { + const ObLSAttr &ls_attr = ls_attr_array.at(i); + if (OB_FAIL(check_ls_attr_state_(ls_attr))) { + LOG_WARN("fail to check ls attr state", KR(ret), K(ls_attr)); + } + } + + return ret; +} + +int ObCollectMvMergeInfoTask::check_ls_list_match(const ObLSAttrArray &ls_attr_array, + const ObLSAttrArray &ls_attr_array_new) +{ + int ret = OB_SUCCESS; + + if (ls_attr_array.count() != ls_attr_array_new.count()) { + ret = OB_ITEM_NOT_MATCH; + LOG_WARN("ls list not match, skip task", KR(ret), + K(ls_attr_array.count()), K(ls_attr_array_new.count())); + } else { + // sorted ls list + ARRAY_FOREACH(ls_attr_array, i) { + const ObLSAttr &ls_attr = ls_attr_array.at(i); + bool is_match = false; + const ObLSAttr &ls_attr_new = ls_attr_array_new.at(i); + if (ls_attr.get_ls_id() == ls_attr_new.get_ls_id()) { + is_match = true; + } else { + ret = OB_ITEM_NOT_MATCH; + LOG_WARN("ls list not match, skip task", KR(ret)); + } + } + } + + return ret; +} + +int ObCollectMvMergeInfoTask::sync_get_ls_member_merge_info(const common::ObAddr &server, + const uint64_t user_tenant_id, + const ObLSID &ls_id, + storage::ObMajorMVMergeInfo &mv_merge_info, + uint64_t rpc_timeout, + const bool need_check_leader, + const bool need_update) +{ + int ret = OB_SUCCESS; + + ObCollectMvMergeInfoArg arg; + if (OB_ISNULL(GCTX.srv_rpc_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("gctx rpc proxy should not be null", K(ret), KP(GCTX.srv_rpc_proxy_)); + } else if (!server.is_valid() || !ls_id.is_valid() || OB_INVALID_TENANT_ID == user_tenant_id) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(server), K(ls_id), K(user_tenant_id)); + } else if (OB_FAIL(arg.init(ls_id, user_tenant_id, need_check_leader, need_update))) { + LOG_WARN("init arg failed", KR(ret)); + } else { + rootserver::ObCollectMvMergeInfoProxy rpc_proxy( + *(GCTX.srv_rpc_proxy_), &obrpc::ObSrvRpcProxy::collect_mv_merge_info); + if (OB_FAIL(rpc_proxy.call(server, rpc_timeout, arg))) { + LOG_WARN("fail to call collect mv merge info rpc", K(ret), K(server), K(arg)); + } + int tmp_ret = OB_SUCCESS; + ObArray return_code_array; + if (OB_TMP_FAIL(rpc_proxy.wait_all(return_code_array))) { + LOG_WARN("fail to wait rpc back", KR(ret), K(server), K(arg)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } else if (OB_FAIL(ret)) { + } else if (OB_FAIL(rpc_proxy.check_return_cnt(return_code_array.count()))) { + LOG_WARN("fail to check return count", KR(ret), K(server), K(arg)); + } else { + const int64_t res_idx = 0; + if (OB_FAIL(return_code_array.at(res_idx))) { + if (ret == OB_TIMEOUT) { + ret = OB_WAIT_NEXT_TIMEOUT; // rewrite ret code + LOG_WARN("rpc return timeout, maybe need retry", KR(ret), K(server), K(arg), K(rpc_timeout)); + } else { + LOG_WARN("rpc return code failed", KR(ret), K(server), K(arg)); + } + } else { + const ObCollectMvMergeInfoResult *res = rpc_proxy.get_results().at(res_idx); + if (OB_ISNULL(res)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is null", KR(ret), K(server), K(arg), KP(res)); + } else if (OB_SUCC(res->get_ret())) { + mv_merge_info = res->get_mv_merge_info(); + } else { + LOG_WARN("fail to get server merge info", KR(ret), K(server), K(arg)); + } + } + } + } + + return ret; +} + +int ObCollectMvMergeInfoTask::collect_ls_member_merge_info(const uint64_t tenant_id, + const ObLSID &ls_id, + share::SCN &merge_scn) +{ + int ret = OB_SUCCESS; + merge_scn.set_max(); + common::ObArray addr_list; + palf::LogConfigVersion log_config_version; + const bool need_check_leader = false; + const bool need_update = true; + if (!ls_id.is_valid() || !is_valid_tenant_id(tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(ls_id), K(tenant_id)); + } else if (OB_ISNULL(GCTX.srv_rpc_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("gctx rpc proxy should not be null", KR(ret), K(ls_id)); + } else if (OB_FAIL(get_stable_member_list_and_config_version(tenant_id, ls_id, + addr_list, log_config_version))) { + LOG_WARN("fail to get member list", KR(ret), K(ls_id)); + } else { + // batch collect ls merge info + ObCollectMvMergeInfoProxy batch_proxy( + *(GCTX.srv_rpc_proxy_), &obrpc::ObSrvRpcProxy::collect_mv_merge_info); + ObCollectMvMergeInfoArg arg; + if (OB_FAIL(arg.init(ls_id, tenant_id, need_check_leader, need_update))) { + LOG_WARN("failed to collect merge info arg", KR(ret), K(arg)); + } else { + ARRAY_FOREACH(addr_list, i) { + const common::ObAddr &addr = addr_list.at(i); + // for debug + LOG_INFO("iter ls memberlist", K(ls_id), K(addr), K(i), K(arg)); + int64_t rpc_timeout = obrpc::ObRpcProxy::MAX_RPC_TIMEOUT; + if (OB_FAIL(batch_proxy.call(addr, rpc_timeout, arg))) { + LOG_WARN("fail to call collect mv merge info rpc", KR(ret)); + } + } + } + + ObArray return_code_array; + // wait all rpc back + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(batch_proxy.wait_all(return_code_array))) { + LOG_WARN("wait batch result failed", KR(ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } else if (OB_FAIL(ret)) { + } else if (OB_FAIL(batch_proxy.check_return_cnt(return_code_array.count()))) { + LOG_WARN("return cnt not match", KR(ret), "return_cnt", return_code_array.count()); + } else { + int64_t min_idx = OB_INVALID_ID; + ARRAY_FOREACH(batch_proxy.get_results(), idx) { + const ObCollectMvMergeInfoResult *res = batch_proxy.get_results().at(idx); + if (OB_ISNULL(res)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result should not be null", KR(ret), KP(res)); + } else if (OB_FAIL(res->get_ret())) { + LOG_WARN("return error code, skip task", KR(ret), KP(res)); + } else if (res->get_mv_merge_scn().is_valid()) { + if (merge_scn > res->get_mv_merge_scn()) { + min_idx = idx; + merge_scn = res->get_mv_merge_scn(); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("collect invalid mv merge scn", KR(ret), K(res->get_mv_merge_scn())); + } + } + LOG_INFO("iter ls memberlist to get merge scn", K(ls_id), K(merge_scn), K(ret), K(min_idx)); + if (OB_SUCC(ret)) { + if (min_idx != OB_INVALID_ID) { + LOG_INFO("iter ls memberlist to get merge scn", K(ls_id), K(min_idx), + K(merge_scn), K(batch_proxy.get_dests().at(min_idx))); + } + common::ObArray addr_list_new; + palf::LogConfigVersion log_config_version_new; + if (OB_FAIL(get_stable_member_list_and_config_version(tenant_id, ls_id, + addr_list_new, log_config_version_new))) { + LOG_WARN("fail to get member list", KR(ret), K(ls_id)); + } else if (log_config_version_new != log_config_version) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("member list or leader changed, skip task", KR(ret), + K(ls_id), K(log_config_version), K(log_config_version_new)); + } + } + } + } + + return ret; +} + +int ObCollectMvMergeInfoTask::double_check_ls_list_in_trans(const ObLSAttrArray &ls_attr_array, + common::ObMySQLTransaction &trans) +{ + int ret = OB_SUCCESS; + + const int64_t timeout = GCONF.internal_sql_execute_timeout; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id_))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant id", KR(ret), K(tenant_id_)); + } else if (!trans.is_started()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("trans is not started", KR(ret)); + } else { + ObLSAttrArray ls_attr_array_new; + share::ObLSAttrOperator ls_attr_operator(tenant_id_, GCTX.sql_proxy_); + if (OB_FAIL(ls_attr_operator.get_all_ls_by_order_in_trans(true /*lock sys ls*/, + ls_attr_array_new, + trans, + true /*only_existing_ls*/))) { + LOG_WARN("fail to get all ls in trans", KR(ret)); + } else if (OB_FAIL(check_ls_list_state(ls_attr_array_new))) { + LOG_WARN("with not normal ls state, skip task", KR(ret)); + } else if (OB_FAIL(check_ls_list_match(ls_attr_array, ls_attr_array_new))) { + LOG_WARN("ls list not match, skip task", KR(ret)); + } + } + + return ret; +} + +int ObCollectMvMergeInfoTask::check_and_update_tenant_merge_scn(const ObLSAttrArray &ls_attr_array, + const share::SCN &merge_scn) +{ + int ret = OB_SUCCESS; + + common::ObMySQLTransaction trans; + if (OB_FAIL(trans.start(GCTX.sql_proxy_, tenant_id_))) { + LOG_WARN("fail to start tans", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(double_check_ls_list_in_trans(ls_attr_array, trans))) { + LOG_WARN("fail to double check ls list", KR(ret), K(tenant_id_)); + } else { + ObGlobalStatProxy proxy(trans, tenant_id_); + share::SCN tmp_merge_scn; + if (OB_FAIL(proxy.get_major_refresh_mv_merge_scn(true /* for_update */, tmp_merge_scn))) { + LOG_WARN("fail to get major_refresh_mv_merge_scn", KR(ret), K(tenant_id_)); + } + if (OB_SUCC(ret)) { + if (!tmp_merge_scn.is_valid() || tmp_merge_scn < merge_scn) { + if (OB_FAIL(proxy.update_major_refresh_mv_merge_scn(merge_scn))) { + LOG_WARN("fail to update major refresh mv merge scn", KR(ret), K(tenant_id_)); + } + } + } + } + + bool is_commit = ret == OB_SUCCESS ? true : false; + int tmp_ret = OB_SUCCESS; + if (trans.is_started()) { + if (OB_TMP_FAIL(trans.end(is_commit))) { + LOG_WARN("fail to end trans", K(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + } + + return ret; +} + +void ObCollectMvMergeInfoTask::runTimerTask() { + int ret = OB_SUCCESS; + + share::SCN merge_scn(share::SCN::max_scn()); + share::ObLSAttrOperator ls_attr_operator(tenant_id_, GCTX.sql_proxy_); + ObLSAttrArray ls_attr_array; + bool need_schedule = false; + share::SCN latest_merge_scn; + share::SCN major_mv_merge_scn; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObCollectMvMergeInfoTask not init", KR(ret), KP(this)); + } else if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql_proxy should not null", KR(ret), KP(GCTX.sql_proxy_)); + } else if (OB_UNLIKELY(is_stop_)) { + // do nothing + } else if (OB_FAIL(need_push_major_mv_merge_scn(tenant_id_, need_schedule, latest_merge_scn, major_mv_merge_scn))) { + LOG_WARN("fail to check need task should schedule or not", KR(ret), K(tenant_id_)); + } + + if (OB_FAIL(ret)) { + } else if (!need_schedule) { + // do nothing + } else if (OB_FAIL(ls_attr_operator.get_all_ls_by_order(ls_attr_array))) { + LOG_WARN("fail to get all ls", KR(ret)); + } else if (OB_FAIL(check_ls_list_state(ls_attr_array))) { + LOG_WARN("with not normal ls state, skip task", KR(ret)); + } else { + share::SCN min_merge_scn(share::SCN::max_scn()); + int64_t min_idx = OB_INVALID_ID; + ARRAY_FOREACH(ls_attr_array, i) { + const ObLSAttr &ls_attr = ls_attr_array.at(i); + if (OB_FAIL(check_ls_attr_state_(ls_attr))) { + LOG_WARN("fail to get all ls", KR(ret), K(ls_attr)); + } else if (ls_attr.get_ls_id().is_sys_ls()) { + // skip sys ls + } else if (OB_FAIL(collect_ls_member_merge_info(tenant_id_, ls_attr.get_ls_id(), merge_scn))) { + LOG_WARN("fail to collect ls member merge scn", KR(ret), K(ls_attr)); + } + if (OB_SUCC(ret)) { + if (min_merge_scn > merge_scn) { + min_idx = i; + min_merge_scn = merge_scn; + } + } + } + // double check ls list and update tenant mv merge scn in trans + if (OB_SUCC(ret) && !min_merge_scn.is_max()) { + if (min_idx != OB_INVALID_ID) { + LOG_INFO("iter all ls and get min merge scn", K(min_idx), K(min_merge_scn), + K(ls_attr_array.at(min_idx))); + } + if (OB_FAIL(check_and_update_tenant_merge_scn(ls_attr_array, min_merge_scn))) { + LOG_WARN("fail to check and update tenant merge scn", KR(ret)); + } + } + } + LOG_INFO("collect_mv_merge_info task", KR(ret), K(need_schedule), K(latest_merge_scn), K(major_mv_merge_scn)); +} + +int ObCollectMvMergeInfoTask::get_min_mv_tablet_major_compaction_scn(share::SCN &compaction_scn) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + const int64_t mv_mode = 1; // bit_map 0x00000001 (32bit), 1 means is new mv tablet + const uint64_t user_tenant_id = gen_user_tenant_id(MTL_ID()); + if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql_proxy should not null", KR(ret), KP(GCTX.sql_proxy_)); + } else if (!is_valid_tenant_id(user_tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant_id is invalid", KR(ret), K(user_tenant_id)); + } else if (OB_FAIL(sql.assign_fmt("SELECT MIN(C.COMPACTION_SCN) AS MIN_COMPACTION_SCN FROM \ + `%s`.`%s` AS A \ + JOIN `%s`.`%s` AS B \ + ON A.TABLE_ID = B.TABLE_ID \ + JOIN `%s`.`%s` AS C \ + ON A.TABLET_ID = C.TABLET_ID \ + JOIN `%s`.`%s` AS D \ + ON C.SVR_IP = D.SVR_IP and C.SVR_PORT = D.SVR_PORT \ + AND A.LS_ID = D.LS_ID \ + WHERE (B.MV_MODE & %ld) <> 0", + OB_SYS_DATABASE_NAME, OB_ALL_TABLET_TO_LS_TNAME, + OB_SYS_DATABASE_NAME, OB_ALL_TABLE_TNAME, + OB_SYS_DATABASE_NAME, OB_ALL_VIRTUAL_TABLET_META_TABLE_TNAME, + OB_SYS_DATABASE_NAME, OB_ALL_VIRTUAL_LOG_STAT_TNAME, + mv_mode))) { + LOG_WARN("fail to format sql", KR(ret), K(sql), K(user_tenant_id)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + sqlclient::ObMySQLResult *mysql_result = NULL; + if (OB_FAIL(GCTX.sql_proxy_->read(res, + user_tenant_id, + sql.ptr()))) { + LOG_WARN("fail to execute sql", K(ret), K(sql), K(user_tenant_id)); + } else if (OB_ISNULL(mysql_result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is null", K(ret), KP(mysql_result)); + } else if (OB_FAIL(mysql_result->next())) { + LOG_WARN("fail to get next", K(ret), KP(mysql_result)); + } else { + uint64_t min_compaction_scn = 0; + EXTRACT_UINT_FIELD_MYSQL(*mysql_result, "MIN_COMPACTION_SCN", min_compaction_scn, uint64_t); + if (OB_SUCC(ret)) { + compaction_scn.convert_for_gts(min_compaction_scn); + } + } + } + // for debug + LOG_INFO("get mv tablet min compaction scn", K(ret), K(compaction_scn), K(sql)); + } + return ret; +} +} +} diff --git a/src/rootserver/mview/ob_collect_mv_merge_info_task.h b/src/rootserver/mview/ob_collect_mv_merge_info_task.h new file mode 100644 index 0000000000..55e3095cb2 --- /dev/null +++ b/src/rootserver/mview/ob_collect_mv_merge_info_task.h @@ -0,0 +1,93 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#ifndef OB_COLLECT_MV_MERGE_INFO_TASK_H +#define OB_COLLECT_MV_MERGE_INFO_TASK_H + +#include "lib/ob_define.h" +#include "share/ob_thread_mgr.h" +#include "rootserver/mview/ob_mview_timer_task.h" +#include "share/ob_srv_rpc_proxy.h" +#include "share/rpc/ob_async_rpc_proxy.h" +#include "rpc/obrpc/ob_rpc_packet.h" +#include "rpc/obrpc/ob_rpc_result_code.h" +#include "rpc/obrpc/ob_rpc_proxy.h" +#include "src/storage/tx_storage/ob_ls_service.h" +#include "src/storage/mview/ob_major_mv_merge_info.h" +#include "lib/hash/ob_hashset.h" + +namespace oceanbase +{ +namespace rootserver +{ + +#define RPC_MVIEW(code, arg, result, name) \ + typedef obrpc::ObAsyncRpcProxy *, const obrpc::ObRpcOpts &), obrpc::ObSrvRpcProxy> name +RPC_MVIEW(obrpc::OB_COLLECT_MV_MERGE_INFO, obrpc::ObCollectMvMergeInfoArg, obrpc::ObCollectMvMergeInfoResult, ObCollectMvMergeInfoProxy); + +typedef common::hash::ObHashSet NewMvBaseTabletSet; + +class ObCollectMvMergeInfoTask : public ObMViewTimerTask +{ +public: + + static const uint64_t SLEEP_SECONDS = 30 * 1000L * 1000L; // 30s + explicit ObCollectMvMergeInfoTask() : is_inited_(false), + is_stop_(true), + in_sched_(false), + tenant_id_(OB_INVALID_TENANT_ID) + {} + ~ObCollectMvMergeInfoTask() { destroy(); } + // static int mtl_init(ObMvMergeInfoCollector *&collector); + int init(); + void reset(); + void destroy(); + int start(); + void stop(); + void wait(); + void runTimerTask(void) override; + static int get_stable_member_list_and_config_version(const uint64_t tenant_id, + const share::ObLSID &ls_id, + common::ObIArray &addr_list, + palf::LogConfigVersion &log_config_version); + int check_ls_list_state(const ObLSAttrArray &ls_attr_array); + int check_ls_list_match(const ObLSAttrArray &ls_attr_array, + const ObLSAttrArray &ls_attr_array_new); + int double_check_ls_list_in_trans(const ObLSAttrArray &ls_attr_array, + common::ObMySQLTransaction &trans); + int check_and_update_tenant_merge_scn(const ObLSAttrArray &ls_attr_array, + const share::SCN &merge_scn); + static int collect_ls_member_merge_info(const uint64_t tenant_id, + const ObLSID &ls_id, + share::SCN &merge_scn); + static int sync_get_ls_member_merge_info(const common::ObAddr &server, + const uint64_t tenant_id, + const ObLSID &ls_id, + storage::ObMajorMVMergeInfo &mv_merge_info, + uint64_t rpc_timeout, + const bool need_check_leader = false, + const bool need_update = false); + static int get_min_mv_tablet_major_compaction_scn(share::SCN &compaction_scn); + TO_STRING_KV(K_(is_inited), K_(is_stop), K_(in_sched), K_(tenant_id)); +private: + int check_ls_attr_state_(const ObLSAttr &ls_attr); +private: + bool is_inited_; + bool is_stop_; + bool in_sched_; + uint64_t tenant_id_; +}; + +} +} + +#endif diff --git a/src/rootserver/mview/ob_mview_clean_snapshot_task.cpp b/src/rootserver/mview/ob_mview_clean_snapshot_task.cpp new file mode 100644 index 0000000000..4f2fed9d91 --- /dev/null +++ b/src/rootserver/mview/ob_mview_clean_snapshot_task.cpp @@ -0,0 +1,252 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX RS + +#include "observer/dbms_scheduler/ob_dbms_sched_job_utils.h" +#include "observer/ob_server_struct.h" +#include "observer/omt/ob_multi_tenant.h" +#include "rootserver/mview/ob_mview_clean_snapshot_task.h" +#include "share/ob_errno.h" +#include "share/schema/ob_mview_info.h" +#include "share/schema/ob_mview_refresh_stats_params.h" +#include "share/ob_global_stat_proxy.h" +#include "share/ob_snapshot_table_proxy.h" +#include "share/inner_table/ob_inner_table_schema.h" +#include "storage/mview/ob_mview_refresh_stats_purge.h" +#include "storage/compaction/ob_tenant_freeze_info_mgr.h" +#include "sql/resolver/mv/ob_mv_dep_utils.h" + +namespace oceanbase { +namespace rootserver { + +ObMViewCleanSnapshotTask::ObMViewCleanSnapshotTask() + : is_inited_(false), + in_sched_(false), + is_stop_(true), + tenant_id_(OB_INVALID_TENANT_ID) +{ +} + +ObMViewCleanSnapshotTask::~ObMViewCleanSnapshotTask() {} + +int ObMViewCleanSnapshotTask::init() +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObMViewCleanSnapshotTask init twice", KR(ret), KP(this)); + } else { + tenant_id_ = MTL_ID(); + is_inited_ = true; + } + return ret; +} + +int ObMViewCleanSnapshotTask::start() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObMViewCleanSnapshotTask not init", KR(ret), KP(this)); + } else { + is_stop_ = false; + if (!in_sched_ && OB_FAIL(schedule_task(MVIEW_CLEAN_SNAPSHOT_INTERVAL, true /*repeat*/))) { + LOG_WARN("fail to schedule ObMViewCleanSnapshotTask", KR(ret)); + } else { + in_sched_ = true; + } + } + return ret; +} + +void ObMViewCleanSnapshotTask::stop() +{ + is_stop_ = true; + in_sched_ = false; + cancel_task(); +} + +void ObMViewCleanSnapshotTask::destroy() +{ + is_inited_ = false; + is_stop_ = true; + in_sched_ = false; + cancel_task(); + wait_task(); + tenant_id_ = OB_INVALID_TENANT_ID; +} + +void ObMViewCleanSnapshotTask::wait() { wait_task(); } + +// This task is used to clean records in __all_acquired_snapshot for the following conditions: +// 1. when a mv is dropped, the snapshot record of its container table should be removed +// 2. when failure occurs in the creation process of a mv, the snapshot record may have been +// created, but the mv is not created successfully, so the snapshot record should be removed +// 3. when multiple mv referring to the same base table, there will be multiple snapshot record for +// the base table, but only the record with smallest scn is necessary, others can be removed +// 4. when all mv of a base table are dropped, the snapshot record of the base table should be +// removed +void ObMViewCleanSnapshotTask::runTimerTask() +{ + int ret = OB_SUCCESS; + uint64_t data_version = 0; + common::ObISQLClient *sql_proxy = GCTX.sql_proxy_; + storage::ObTenantFreezeInfoMgr *mgr = MTL(storage::ObTenantFreezeInfoMgr *); + bool need_schedule = false; + ObMySQLTransaction trans; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObMViewCleanSnapshotTask not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(is_stop_)) { + + } else if (OB_FAIL(need_schedule_major_refresh_mv_task(tenant_id_, need_schedule))) { + LOG_WARN("fail to check need schedule major refresh mv task", KR(ret), K(tenant_id_)); + } else if (!need_schedule) { + + } else if (OB_UNLIKELY(OB_ISNULL(sql_proxy) || OB_ISNULL(mgr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql_proxy or mgr is null", KR(ret), K(sql_proxy), K(mgr)); + } else if (OB_FAIL(trans.start(sql_proxy, tenant_id_))) { + LOG_WARN("fail to start trans", KR(ret), K(tenant_id_)); + } else { + share::ObGlobalStatProxy stat_proxy(trans, tenant_id_); + share::SCN major_refresh_mv_merge_scn; + share::ObSnapshotTableProxy snapshot_proxy; + const bool select_for_update = true; + ObSEArray snapshots; + bool mview_in_creation = false; + if (OB_FAIL(stat_proxy.get_major_refresh_mv_merge_scn(select_for_update, + major_refresh_mv_merge_scn))) { + LOG_WARN("fail to get major_refresh_mv_merge_scn", KR(ret), K(tenant_id_)); + } else if (OB_UNLIKELY(!major_refresh_mv_merge_scn.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("major_refresh_mv_merge_scn is invalid", KR(ret), K(tenant_id_), + K(major_refresh_mv_merge_scn)); + } else if (OB_FAIL(snapshot_proxy.get_all_snapshots( + trans, tenant_id_, share::SNAPSHOT_FOR_MAJOR_REFRESH_MV, snapshots))) { + LOG_WARN("fail to get snapshots", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(ObMViewInfo::contains_major_refresh_mview_in_creation( + trans, tenant_id_, mview_in_creation))) { + LOG_WARN("fail to check if mv is in creation", KR(ret), K(tenant_id_)); + } else if (mview_in_creation) { + // when a mview is being created, its snapshot may be added but the dependency is not + // added yet, which can cause cleaning the snapshot by mistake. so we just skip this round. + LOG_INFO("mview is being created, skip clean task", KR(ret), K(tenant_id_)); + } else { + uint64_t last_tablet_id = 0; + // considering the task is scheduled infrequently, performance should not be a concern here, + // so we use a simple loop to remove the snapshots for now. + for (int64_t i = 0; OB_SUCC(ret) && i < snapshots.count(); ++i) { + const share::ObSnapshotInfo &snapshot = snapshots.at(i); + uint64_t table_id = 0; + ObSEArray relevent_mv_tables; + bool is_container_table = false; + bool need_remove_snapshot = false; + if (OB_UNLIKELY(!snapshot.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("snapshot is invalid", KR(ret), K(snapshot)); + } else if (snapshot.tablet_id_ == last_tablet_id) { + // the snapshots list is sorted by tablet_id, snapshot_scn, + // so if the tablet_id is the same as the last one, we can safely remove this snapshot. + need_remove_snapshot = true; + LOG_INFO("redundant snapshot, remove snapshot", KR(ret), K(tenant_id_), K(snapshot)); + } else if (FALSE_IT(last_tablet_id = snapshot.tablet_id_)) { + } else if (OB_FAIL(get_table_id_(trans, snapshot.tablet_id_, table_id))) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + need_remove_snapshot = true; + LOG_INFO("table is dropped, remove snapshot", KR(ret), K(tenant_id_), K(snapshot)); + } else { + LOG_WARN("fail to get table id", KR(ret), K(snapshot)); + } + } else if (OB_FAIL(is_mv_container_table_(table_id, is_container_table))) { + LOG_WARN("fail to check is mv container table", KR(ret), K(tenant_id_), K(table_id)); + } else if (is_container_table) { + // do nothing + } else if (OB_FAIL(sql::ObMVDepUtils::get_referring_mv_of_base_table( + trans, tenant_id_, table_id, relevent_mv_tables))) { + LOG_WARN("fail to get referring mv of base table", KR(ret), K(tenant_id_), K(table_id)); + } else if (relevent_mv_tables.empty()) { + need_remove_snapshot = true; + LOG_INFO("no relevant mv, remove snapshot", KR(ret), K(tenant_id_), K(snapshot)); + } + if (OB_FAIL(ret) || !need_remove_snapshot) { + // do nothing + } else if (OB_FAIL(snapshot_proxy.remove_snapshot(trans, tenant_id_, snapshot))) { + LOG_WARN("fail to remove snapshot", KR(ret), K(tenant_id_), K(snapshot)); + } else { + LOG_INFO("[MAJ_REF_MV] successfully remove snapshot", KR(ret), K(tenant_id_), K(snapshot)); + } + } + } + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + } + } +} + +int ObMViewCleanSnapshotTask::get_table_id_(ObISQLClient &sql_client, const uint64_t tablet_id, + uint64_t &table_id) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + + SMART_VAR(ObMySQLProxy::MySQLResult, res) + { + common::sqlclient::ObMySQLResult *result = nullptr; + if (OB_FAIL(sql.assign_fmt("SELECT table_id FROM %s WHERE tablet_id = %ld", + share::OB_ALL_TABLET_TO_LS_TNAME, tablet_id))) { + LOG_WARN("fail to assign sql", KR(ret)); + } else if (OB_FAIL(sql_client.read(res, tenant_id_, sql.ptr()))) { + LOG_WARN("execute sql failed", KR(ret), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is null", KR(ret)); + } else if (OB_FAIL(result->next())) { + LOG_WARN("fail to get next", KR(ret)); + } else { + EXTRACT_INT_FIELD_MYSQL(*result, "table_id", table_id, uint64_t); + } + } + + return ret; +} + +int ObMViewCleanSnapshotTask::is_mv_container_table_(const uint64_t table_id, bool &is_container) +{ + int ret = OB_SUCCESS; + ObMultiVersionSchemaService *schema_service = GCTX.schema_service_; + ObSchemaGetterGuard schema_guard; + const ObTableSchema *table_schema = NULL; + is_container = false; + + if (OB_FAIL(schema_service->get_tenant_schema_guard(tenant_id_, schema_guard))) { + LOG_WARN("get tenant schema guard failed", K(ret), K(tenant_id_)); + } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id_, table_id, table_schema))) { + LOG_WARN("failed to get table schema", KR(ret), K(tenant_id_), K(table_id)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("table schema is null", KR(ret), K(tenant_id_), K(table_id)); + } else { + is_container = table_schema->mv_major_refresh(); + } + + return ret; +} +} // namespace rootserver +} // namespace oceanbase diff --git a/src/rootserver/mview/ob_mview_clean_snapshot_task.h b/src/rootserver/mview/ob_mview_clean_snapshot_task.h new file mode 100644 index 0000000000..21a9c4e8b2 --- /dev/null +++ b/src/rootserver/mview/ob_mview_clean_snapshot_task.h @@ -0,0 +1,50 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#pragma once + +#include "lib/task/ob_timer.h" +#include "lib/mysqlclient/ob_isql_client.h" +#include "rootserver/mview/ob_mview_timer_task.h" + +namespace oceanbase +{ +namespace rootserver +{ +class ObMViewCleanSnapshotTask : public ObMViewTimerTask +{ +public: + ObMViewCleanSnapshotTask(); + virtual ~ObMViewCleanSnapshotTask(); + DISABLE_COPY_ASSIGN(ObMViewCleanSnapshotTask); + // for Service + int init(); + int start(); + void stop(); + void wait(); + void destroy(); + // for TimerTask + void runTimerTask() override; + // TODO: increase the scheduling interval + static const int64_t MVIEW_CLEAN_SNAPSHOT_INTERVAL = 60 * 1000 * 1000; // 1min +private: + int get_table_id_(ObISQLClient &sql_client, const uint64_t tablet_id, uint64_t &table_id); + int is_mv_container_table_(const uint64_t table_id, bool &is_container); +private: + bool is_inited_; + bool in_sched_; + bool is_stop_; + uint64_t tenant_id_; +}; + +} // namespace rootserver +} // namespace oceanbase diff --git a/src/rootserver/mview/ob_mview_dependency_service.cpp b/src/rootserver/mview/ob_mview_dependency_service.cpp index c617c041a2..6f430f3b96 100644 --- a/src/rootserver/mview/ob_mview_dependency_service.cpp +++ b/src/rootserver/mview/ob_mview_dependency_service.cpp @@ -15,6 +15,7 @@ #include "share/schema/ob_multi_version_schema_service.h" #include "share/schema/ob_schema_getter_guard.h" #include "share/schema/ob_table_sql_service.h" +#include "share/schema/ob_mview_info.h" #include "sql/resolver/mv/ob_mv_dep_utils.h" namespace oceanbase @@ -43,6 +44,7 @@ int ObMViewDependencyService::remove_mview_dep_infos( { int ret = OB_SUCCESS; ObArray stale_ref_table_ids; + ObArray stale_fast_lsm_ref_table_ids; // during upgrading, dropping mview should still work, // hence, do nothing if __all_mview_dep does not exists bool all_mview_dep_table_exists = false; @@ -54,14 +56,30 @@ int ObMViewDependencyService::remove_mview_dep_infos( trans, tenant_id, mview_table_id, stale_ref_table_ids))) { LOG_WARN("failed to get table ids only referenced by given mv", KR(ret)); } else if (!stale_ref_table_ids.empty()) { - enum ObTableReferencedByMVFlag table_ref_by_mv_flag = - ObTableReferencedByMVFlag::IS_NOT_REFERENCED_BY_MV; + ObUpdateMViewRefTableOpt opt; + opt.set_table_flag(ObTableReferencedByMVFlag::IS_NOT_REFERENCED_BY_MV); + opt.set_mv_flag(ObTableReferencedByFastLSMMVFlag::IS_NOT_REFERENCED_BY_FAST_LSM_MV); if (OB_FAIL(update_mview_reference_table_status(trans, schema_guard, tenant_id, stale_ref_table_ids, - table_ref_by_mv_flag))) { - LOG_WARN("failed to update mview reference table status", KR(ret)); + opt))) { + LOG_WARN("failed to update mview reference table status", KR(ret), K(tenant_id), + K(stale_ref_table_ids), K(opt)); + } + } else if (OB_FAIL(ObMVDepUtils::get_table_ids_only_referenced_by_given_fast_lsm_mv( + trans, tenant_id, mview_table_id, stale_fast_lsm_ref_table_ids))) { + LOG_WARN("failed to get table ids only referenced by given fast lsm mv", KR(ret)); + } else if (!stale_fast_lsm_ref_table_ids.empty()) { + ObUpdateMViewRefTableOpt opt; + opt.set_mv_flag(ObTableReferencedByFastLSMMVFlag::IS_NOT_REFERENCED_BY_FAST_LSM_MV); + if (OB_FAIL(update_mview_reference_table_status(trans, + schema_guard, + tenant_id, + stale_fast_lsm_ref_table_ids, + opt))) { + LOG_WARN("failed to update mview reference table status", KR(ret), K(tenant_id), + K(stale_fast_lsm_ref_table_ids), K(opt)); } } if (OB_SUCC(ret) && OB_FAIL(sql::ObMVDepUtils::delete_mview_dep_infos( @@ -84,7 +102,10 @@ int ObMViewDependencyService::update_mview_dep_infos( ObArray prev_mv_dep_infos; ObArray new_ref_table_ids;// table_referenced_by_mv_flag will be set ObArray stale_ref_table_ids; // table_referenced_by_mv_flag will be cleared + ObArray stale_fast_lsm_ref_table_ids; ObArray table_ids_only_ref_by_this_mv; + ObArray table_ids_only_ref_by_this_fast_lsm_mv; + ObMViewInfo mview_info; // during upgrading, creating mview should still work, // hence, do nothing if __all_mview_dep does not exists bool all_mview_dep_table_exists = false; @@ -96,7 +117,10 @@ int ObMViewDependencyService::update_mview_dep_infos( trans, tenant_id, OB_ALL_MVIEW_DEP_TID, all_mview_dep_table_exists))) { LOG_WARN("failed to check is system table name", KR(ret)); } else if (all_mview_dep_table_exists) { - if (OB_FAIL(sql::ObMVDepUtils::convert_to_mview_dep_infos(dep_infos, cur_mv_dep_infos))) { + if (OB_FAIL(ObMViewInfo::fetch_mview_info(trans, tenant_id, mview_table_id, mview_info, + false /*for_update*/, true /*nowait*/))) { + LOG_WARN("fail to fetch mview info", KR(ret), K(mview_table_id)); + } else if (OB_FAIL(sql::ObMVDepUtils::convert_to_mview_dep_infos(dep_infos, cur_mv_dep_infos))) { LOG_WARN("failed to convert to mview dep infos", KR(ret)); } else if (OB_FAIL(sql::ObMVDepUtils::get_mview_dep_infos( trans, tenant_id, mview_table_id, prev_mv_dep_infos))) { @@ -104,6 +128,9 @@ int ObMViewDependencyService::update_mview_dep_infos( } else if (OB_FAIL(ObMVDepUtils::get_table_ids_only_referenced_by_given_mv( trans, tenant_id, mview_table_id, table_ids_only_ref_by_this_mv))) { LOG_WARN("failed to get table ids only referenced by given mv", KR(ret)); + } else if (OB_FAIL(ObMVDepUtils::get_table_ids_only_referenced_by_given_fast_lsm_mv( + trans, tenant_id, mview_table_id, table_ids_only_ref_by_this_fast_lsm_mv))) { + LOG_WARN("failed to get table ids only referenced by given fast lsm mv", KR(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && (i < cur_mv_dep_infos.count()); ++i) { const ObMVDepInfo &cur_mv_dep = cur_mv_dep_infos.at(i); @@ -132,11 +159,15 @@ int ObMViewDependencyService::update_mview_dep_infos( // if an old_ref_table_id exists in the cur_mv_dep_infos, // then its table_referenced_by_mv_flag does not need to be cleared if (has_exist_in_array(new_ref_table_ids, old_ref_table_id)) { - } else if (has_exist_in_array(table_ids_only_ref_by_this_mv, prev_mv_dep.p_obj_)) { + } else if (has_exist_in_array(table_ids_only_ref_by_this_mv, old_ref_table_id)) { // only when an old_ref_table_id exists in the list of table_ids_only_ref_by_this_mv, // its table_referenced_by_mv_flag needs to be cleared - if (OB_FAIL(stale_ref_table_ids.push_back(prev_mv_dep.p_obj_))) { - LOG_WARN("failed to add old ref table id to array", KR(ret), K(prev_mv_dep.p_obj_)); + if (OB_FAIL(stale_ref_table_ids.push_back(old_ref_table_id))) { + LOG_WARN("failed to add old ref table id to array", KR(ret), K(old_ref_table_id)); + } + } else if (has_exist_in_array(table_ids_only_ref_by_this_fast_lsm_mv, old_ref_table_id)) { + if (OB_FAIL(stale_fast_lsm_ref_table_ids.push_back(old_ref_table_id))) { + LOG_WARN("failed to add old ref table id to array", KR(ret), K(old_ref_table_id)); } } } @@ -144,20 +175,38 @@ int ObMViewDependencyService::update_mview_dep_infos( } if (OB_SUCC(ret) && !stale_ref_table_ids.empty()) { - enum ObTableReferencedByMVFlag table_ref_by_mv_flag = - ObTableReferencedByMVFlag::IS_NOT_REFERENCED_BY_MV; + ObUpdateMViewRefTableOpt opt; + opt.set_table_flag(ObTableReferencedByMVFlag::IS_NOT_REFERENCED_BY_MV); + opt.set_mv_flag(ObTableReferencedByFastLSMMVFlag::IS_NOT_REFERENCED_BY_FAST_LSM_MV); if (OB_FAIL(update_mview_reference_table_status(trans, schema_guard, tenant_id, stale_ref_table_ids, - table_ref_by_mv_flag))) { - LOG_WARN("failed to update mview reference table status", KR(ret)); + opt))) { + LOG_WARN("failed to update mview reference table status", KR(ret), K(tenant_id), + K(stale_ref_table_ids), K(opt)); + } + } + + if (OB_SUCC(ret) && !stale_fast_lsm_ref_table_ids.empty()) { + ObUpdateMViewRefTableOpt opt; + opt.set_mv_flag(ObTableReferencedByFastLSMMVFlag::IS_NOT_REFERENCED_BY_FAST_LSM_MV); + if (OB_FAIL(update_mview_reference_table_status(trans, + schema_guard, + tenant_id, + stale_fast_lsm_ref_table_ids, + opt))) { + LOG_WARN("failed to update mview reference table status", KR(ret), K(tenant_id), + K(stale_fast_lsm_ref_table_ids), K(opt)); } } if (OB_SUCC(ret) && !new_ref_table_ids.empty()) { - enum ObTableReferencedByMVFlag table_ref_by_mv_flag = - ObTableReferencedByMVFlag::IS_REFERENCED_BY_MV; + ObUpdateMViewRefTableOpt opt; + opt.set_table_flag(ObTableReferencedByMVFlag::IS_REFERENCED_BY_MV); + if (mview_info.is_fast_lsm_mv()) { + opt.set_mv_flag(ObTableReferencedByFastLSMMVFlag::IS_REFERENCED_BY_FAST_LSM_MV); + } if (OB_FAIL(sql::ObMVDepUtils::delete_mview_dep_infos( trans, tenant_id, mview_table_id))) { LOG_WARN("failed to delete mview dep infos", KR(ret), K(mview_table_id)); @@ -168,9 +217,8 @@ int ObMViewDependencyService::update_mview_dep_infos( schema_guard, tenant_id, new_ref_table_ids, - table_ref_by_mv_flag))) { - LOG_WARN("failed to update mview reference table status", - KR(ret), K(table_ref_by_mv_flag )); + opt))) { + LOG_WARN("failed to update mview reference table status", KR(ret), K(opt)); } } } @@ -183,15 +231,11 @@ int ObMViewDependencyService::update_mview_reference_table_status( ObSchemaGetterGuard &schema_guard, const uint64_t tenant_id, const ObIArray &ref_table_ids, - enum ObTableReferencedByMVFlag table_flag) + const ObUpdateMViewRefTableOpt &update_opt) { int ret = OB_SUCCESS; uint64_t compat_version = 0; - if ((ObTableReferencedByMVFlag::IS_REFERENCED_BY_MV != table_flag) - && (ObTableReferencedByMVFlag::IS_NOT_REFERENCED_BY_MV != table_flag)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid table_flag", KR(ret), K(table_flag)); - } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { LOG_WARN("failed to get data version", KR(ret), K(tenant_id)); } else if (compat_version < DATA_VERSION_4_3_1_0) { ret = OB_NOT_SUPPORTED; @@ -210,9 +254,6 @@ int ObMViewDependencyService::update_mview_reference_table_status( } else if (OB_ISNULL(ref_table_schema)) { // the reference table has already been dropped, ignore it LOG_TRACE("ref table schema is null", KR(ret), K(tenant_id), K(ref_table_id)); - } else if (table_flag == ObTableMode::get_table_referenced_by_mv_flag( - ref_table_schema->get_table_mode())) { - // bypass } else if (OB_FAIL(schema_service_.gen_new_schema_version(tenant_id, new_schema_version))) { LOG_WARN("fail to gen new schema_version", KR(ret), K(tenant_id)); } else { @@ -221,12 +262,17 @@ int ObMViewDependencyService::update_mview_reference_table_status( LOG_WARN("fail to assign ref table schema", KR(ret)); } else { new_ref_table_schema.set_table_id(ref_table_id); - new_ref_table_schema.set_table_referenced_by_mv(table_flag); new_ref_table_schema.set_schema_version(new_schema_version); + if (update_opt.need_update_table_flag_) { + new_ref_table_schema.set_table_referenced_by_mv(update_opt.table_flag_); + } + if (update_opt.need_update_mv_flag_) { + new_ref_table_schema.set_table_referenced_by_fast_lsm_mv(update_opt.mv_flag_); + } if (OB_FAIL(schema_service->get_table_sql_service().update_mview_reference_table_status( new_ref_table_schema, trans))) { - LOG_WARN("failed to update mview reference table status", - KR(ret), K(ref_table_id), K(table_flag)); + LOG_WARN("failed to update mview reference table status", KR(ret), K(ref_table_id), + K(update_opt)); } } } diff --git a/src/rootserver/mview/ob_mview_dependency_service.h b/src/rootserver/mview/ob_mview_dependency_service.h index f0b12d453a..c1c05170ff 100644 --- a/src/rootserver/mview/ob_mview_dependency_service.h +++ b/src/rootserver/mview/ob_mview_dependency_service.h @@ -34,6 +34,32 @@ namespace schema } namespace rootserver { +class ObUpdateMViewRefTableOpt +{ + friend class ObMViewDependencyService; +public: + ObUpdateMViewRefTableOpt() : need_update_table_flag_(false), need_update_mv_flag_(false) {} + ~ObUpdateMViewRefTableOpt() {} + void set_table_flag(const share::schema::ObTableReferencedByMVFlag &table_flag) + { + table_flag_ = table_flag; + need_update_table_flag_ = true; + } + void set_mv_flag(const share::schema::ObTableReferencedByFastLSMMVFlag &mv_flag) + { + mv_flag_ = mv_flag; + need_update_mv_flag_ = true; + } + TO_STRING_KV("need_update_table_flag_", need_update_table_flag_, + "table_flag", table_flag_, + "need_update_mv_flag_", need_update_mv_flag_, + "mv_flag", mv_flag_); +private: + bool need_update_table_flag_; + share::schema::ObTableReferencedByMVFlag table_flag_; + bool need_update_mv_flag_; + share::schema::ObTableReferencedByFastLSMMVFlag mv_flag_; +}; class ObMViewDependencyService { public: @@ -53,7 +79,7 @@ public: share::schema::ObSchemaGetterGuard &schema_guard, const uint64_t tenant_id, const ObIArray &ref_table_ids, - enum share::schema::ObTableReferencedByMVFlag table_flags); + const ObUpdateMViewRefTableOpt &update_opt); private: share::schema::ObMultiVersionSchemaService &schema_service_; }; diff --git a/src/rootserver/mview/ob_mview_maintenance_service.cpp b/src/rootserver/mview/ob_mview_maintenance_service.cpp index 7721ff149e..135fde3dd8 100644 --- a/src/rootserver/mview/ob_mview_maintenance_service.cpp +++ b/src/rootserver/mview/ob_mview_maintenance_service.cpp @@ -16,13 +16,13 @@ #include "observer/omt/ob_multi_tenant.h" #include "share/ob_errno.h" #include "share/rc/ob_tenant_base.h" +#include "share/schema/ob_schema_struct.h" // ObMvRefreshMode namespace oceanbase { namespace rootserver { using namespace common; - /** * ObMViewMaintenanceService */ @@ -46,9 +46,16 @@ int ObMViewMaintenanceService::mtl_init(ObMViewMaintenanceService *&service) int ObMViewMaintenanceService::init() { int ret = OB_SUCCESS; + const uint64_t tenant_id = MTL_ID(); + const uint64_t bucket_num = 64; + ObMemAttr attr(tenant_id, "MvRefreshInfo"); + ObMemAttr cache_attr(tenant_id, "MvCacheTask"); if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("ObMViewMaintenanceService init twice", KR(ret), KP(this)); + } else if (!is_valid_tenant_id(tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant id", KR(ret), K(tenant_id)); } else { if (OB_FAIL(mlog_maintenance_task_.init())) { LOG_WARN("fail to init mlog maintenance task", KR(ret)); @@ -56,6 +63,20 @@ int ObMViewMaintenanceService::init() LOG_WARN("fail to init mview maintenance task", KR(ret)); } else if (OB_FAIL(mvref_stats_maintenance_task_.init())) { LOG_WARN("fail to init mvref stats maintenance task", KR(ret)); + } else if (OB_FAIL(mview_push_refresh_scn_task_.init())) { + LOG_WARN("fail to init mview push refresh scn task", KR(ret)); + } else if (OB_FAIL(mview_push_snapshot_task_.init())) { + LOG_WARN("fail to init mview push snapshot task", KR(ret)); + } else if (OB_FAIL(replica_safe_check_task_.init())) { + LOG_WARN("fail to init mvref stats maintenance task", KR(ret)); + } else if (OB_FAIL(collect_mv_merge_info_task_.init())) { + LOG_WARN("collect mv merge info task init failed", KR(ret)); + } else if (OB_FAIL(mview_clean_snapshot_task_.init())) { + LOG_WARN("fail to init mview clean snapshot task", KR(ret)); + } else if (OB_FAIL(mview_update_cache_task_.init())) { + LOG_WARN("fail to init mview update cache task", KR(ret)); + } else if (OB_FAIL(mview_refresh_info_map_.create(bucket_num, attr))) { + LOG_WARN("fail to create mview refresh info map", KR(ret)); } else { is_inited_ = true; } @@ -80,6 +101,11 @@ void ObMViewMaintenanceService::stop() mlog_maintenance_task_.stop(); mview_maintenance_task_.stop(); mvref_stats_maintenance_task_.stop(); + mview_push_refresh_scn_task_.stop(); + mview_push_snapshot_task_.stop(); + replica_safe_check_task_.stop(); + collect_mv_merge_info_task_.stop(); + mview_clean_snapshot_task_.stop(); } void ObMViewMaintenanceService::wait() @@ -87,6 +113,12 @@ void ObMViewMaintenanceService::wait() mlog_maintenance_task_.wait(); mview_maintenance_task_.wait(); mvref_stats_maintenance_task_.wait(); + mview_push_refresh_scn_task_.wait(); + mview_push_snapshot_task_.wait(); + replica_safe_check_task_.wait(); + collect_mv_merge_info_task_.wait(); + mview_clean_snapshot_task_.wait(); + mview_update_cache_task_.wait(); } void ObMViewMaintenanceService::destroy() @@ -95,6 +127,13 @@ void ObMViewMaintenanceService::destroy() mlog_maintenance_task_.destroy(); mview_maintenance_task_.destroy(); mvref_stats_maintenance_task_.destroy(); + mview_push_refresh_scn_task_.destroy(); + mview_push_snapshot_task_.destroy(); + replica_safe_check_task_.destroy(); + collect_mv_merge_info_task_.destroy(); + mview_clean_snapshot_task_.destroy(); + mview_update_cache_task_.destroy(); + mview_refresh_info_map_.destroy(); } int ObMViewMaintenanceService::inner_switch_to_leader() @@ -112,6 +151,18 @@ int ObMViewMaintenanceService::inner_switch_to_leader() LOG_WARN("fail to start mview maintenance task", KR(ret)); } else if (OB_FAIL(mvref_stats_maintenance_task_.start())) { LOG_WARN("fail to start mvref stats maintenance task", KR(ret)); + } else if (OB_FAIL(mview_push_refresh_scn_task_.start())) { + LOG_WARN("fail to start mview push refresh scn task", KR(ret)); + } else if (OB_FAIL(mview_push_snapshot_task_.start())) { + LOG_WARN("fail to start mview push snapshot task", KR(ret)); + } else if (OB_FAIL(replica_safe_check_task_.start())) { + LOG_WARN("fail to start mvref stats maintenance task", KR(ret)); + } else if (OB_FAIL(collect_mv_merge_info_task_.start())) { + LOG_WARN("collect mv merge info task start failed", KR(ret)); + } else if (OB_FAIL(mview_clean_snapshot_task_.start())) { + LOG_WARN("fail to start mview clean snapshot task", KR(ret)); + } else if (OB_FAIL(mview_update_cache_task_.start())) { + LOG_WARN("fail to start mview update cache task", KR(ret)); } } const int64_t cost_us = ObTimeUtility::current_time() - start_time_us; @@ -128,7 +179,11 @@ int ObMViewMaintenanceService::inner_switch_to_follower() ret = OB_NOT_INIT; LOG_WARN("ObMViewMaintenanceService not init", KR(ret), KP(this)); } else { - mvref_stats_maintenance_task_.stop(); + // start update cache task in follower + if (OB_FAIL(mview_update_cache_task_.start())) { + LOG_WARN("fail to start mview update cache task", KR(ret)); + } + stop(); } const int64_t cost_us = ObTimeUtility::current_time() - start_time_us; FLOG_INFO("mview_maintenance: switch_to_follower", KR(ret), K(tenant_id), K(cost_us)); @@ -170,5 +225,207 @@ int ObMViewMaintenanceService::resume_leader() return ret; } +int ObMViewMaintenanceService::extract_sql_result(sqlclient::ObMySQLResult *mysql_result, + ObIArray &mview_ids, + ObIArray &last_refresh_scns, + ObIArray &mview_refresh_modes) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(mysql_result)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mysql result is null", K(ret), KP(mysql_result)); + } else { + ObSEArray res_ids; + ObSEArray res_scns; + ObSEArray refresh_modes; + const int64_t col_idx0 = 0; + const int64_t col_idx1 = 1; + const int64_t col_idx2 = 2; + while (OB_SUCC(ret) && OB_SUCC(mysql_result->next())) { + uint64_t mview_id = OB_INVALID_ID; + uint64_t last_refresh_scn = OB_INVALID_SCN_VAL; + uint64_t refresh_mode = (uint64_t)ObMVRefreshMode::MAX; + if (OB_FAIL(mysql_result->get_uint(col_idx0, mview_id)) + || OB_FAIL(mysql_result->get_uint(col_idx1, last_refresh_scn)) + || OB_FAIL(mysql_result->get_uint(col_idx2, refresh_mode))) { + LOG_WARN("fail to get int/uint value", K(ret)); + } else if (OB_FAIL(res_ids.push_back(mview_id)) + || OB_FAIL(res_scns.push_back(last_refresh_scn)) + || OB_FAIL(refresh_modes.push_back(refresh_mode))) { + LOG_WARN("fail to push back array", K(ret)); + } + } + if (OB_LIKELY(OB_SUCCESS == ret || OB_ITER_END == ret)) { + if((OB_FAIL(mview_ids.assign(res_ids)) || + OB_FAIL(last_refresh_scns.assign(res_scns)) || + OB_FAIL(mview_refresh_modes.assign(refresh_modes)))) { + LOG_WARN("fail to assign array", K(ret)); + } + } + } + return ret; +} + +int ObMViewMaintenanceService::update_mview_refresh_info_cache( + const ObIArray &mview_ids, + const ObIArray &mview_refresh_scns, + const ObIArray &mview_refresh_modes, + ObMviewRefreshInfoMap &mview_refresh_info_map) { + int ret = OB_SUCCESS; + int update_cache_cnt = 0; + const int invalid_refresh_scn = 0; + ARRAY_FOREACH_X(mview_ids, idx, cnt, OB_SUCC(ret)) { + RefreshInfo new_refresh_info; + if (mview_refresh_scns.at(idx) == invalid_refresh_scn) { + // skip update invalid scn in cache + } else if (mview_refresh_modes.at(idx) == (uint64_t)ObMVRefreshMode::MAJOR_COMPACTION) { + new_refresh_info.refresh_scn_ = mview_refresh_scns.at(idx); + new_refresh_info.refresh_ts_ = ObTimeUtility::fast_current_time(); + new_refresh_info.expired_ts_ = new_refresh_info.refresh_ts_ + + ObMViewMaintenanceService::CacheValidInterval; + if (OB_FAIL(mview_refresh_info_map.set_refactored(mview_ids.at(idx), new_refresh_info, 1/*overwrite*/))) { + LOG_WARN("fail to set refresh info", KR(ret), K(idx), K(mview_ids.at(idx))); + } + update_cache_cnt += 1; + // for debug + LOG_INFO("update mview refresh info", K(ret), K(mview_refresh_scns.at(idx)), + K(update_cache_cnt)); + } + } + return ret; +} + +int ObMViewMaintenanceService:: + get_mview_last_refresh_info(const ObIArray &src_mview_ids, + ObMySQLProxy *sql_proxy, + const uint64_t tenant_id, + const share::SCN &scn, + ObIArray &mview_ids, + ObIArray &last_refresh_scns, + ObIArray &mview_refresh_modes) +{ + int ret = OB_SUCCESS; + mview_ids.reuse(); + last_refresh_scns.reuse(); + mview_refresh_modes.reuse(); + if (OB_ISNULL(sql_proxy)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + ObSqlString sql; + sqlclient::ObMySQLResult *mysql_result = NULL; + if (OB_FAIL(sql::ObExprLastRefreshScn:: + get_last_refresh_scn_sql(scn, src_mview_ids, sql))) { + LOG_WARN("failed to get last refresh scn sql", K(ret), K(sql)); + } else if (OB_FAIL(sql_proxy->read(res, + tenant_id, + sql.ptr()))) { + LOG_WARN("fail to execute sql", K(ret), K(sql), K(tenant_id)); + } else if (OB_FAIL(extract_sql_result(res.get_result(), + mview_ids, + last_refresh_scns, + mview_refresh_modes))) { + LOG_WARN("failt to extract sql result", K(ret), K(sql), K(tenant_id)); + } + } + } + return ret; +} + +int ObMViewMaintenanceService::fetch_mv_refresh_scns( + const ObIArray &src_mview_ids, + const share::SCN &read_snapshot, + ObIArray &mview_ids, + ObIArray &mview_refresh_scns, + uint64_t ¬_hit_count) +{ + int ret = OB_SUCCESS; + if (src_mview_ids.empty()) { + // do nothing + } else if (!read_snapshot.is_valid()) { + not_hit_count += 1; + } else { + set_last_request_ts(ObTimeUtility::fast_current_time()); + ARRAY_FOREACH_X(src_mview_ids, idx, cnt, OB_SUCC(ret) && 0 == not_hit_count) { + RefreshInfo refresh_info; + if (OB_FAIL(mview_refresh_info_map_.get_refactored(src_mview_ids.at(idx), refresh_info))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + not_hit_count += 1; + } + LOG_WARN("fail to get refresh info", KR(ret), K(idx), K(src_mview_ids.at(idx))); + } else { + if (refresh_info.hit_cache(read_snapshot)) { + if (OB_FAIL(mview_refresh_scns.push_back(refresh_info.refresh_scn_))) { + LOG_WARN("fail to push back refresh scns", KR(ret), K(idx), K(src_mview_ids.at(idx))); + } + } else { + not_hit_count += 1; + } + } + } + } + return ret; +} + +int ObMViewMaintenanceService::get_mview_refresh_info(const ObIArray &src_mview_ids, + ObMySQLProxy *sql_proxy, + const share::SCN &read_snapshot, + ObIArray &mview_ids, + ObIArray &mview_refresh_scns) +{ + int ret = OB_SUCCESS; + uint64_t not_hit_count = 0; + const uint64_t tenant_id = MTL_ID(); + ObSEArray refresh_modes; + ObSEArray refresh_scns; + if (!is_inited_ || !mview_refresh_info_map_.created()) { + ret = OB_NOT_INIT; + LOG_WARN("ObMViewMaintenanceService not init", KR(ret), + K(is_inited_), K(mview_refresh_info_map_.created())); + } else if (OB_ISNULL(sql_proxy)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", KR(ret), KP(sql_proxy)); + } else if (!is_valid_tenant_id(tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant id", KR(ret), K(tenant_id)); + } else if (src_mview_ids.empty()) { + // do nothing + } else if (OB_FAIL(fetch_mv_refresh_scns(src_mview_ids, read_snapshot, + mview_ids, refresh_scns, not_hit_count))){ + LOG_WARN("fail to fetch mv refresh scns", KR(ret), K(tenant_id), K(src_mview_ids)); + } + if (OB_FAIL(ret)) { + } else if (not_hit_count == 0) { + if (OB_FAIL(mview_ids.assign(src_mview_ids)) || + OB_FAIL(mview_refresh_scns.assign(refresh_scns))) { + LOG_WARN("fail to assign mview ids or mview refresh scns", K(ret)); + } + } else { + mview_refresh_scns.reuse(); + if (OB_FAIL(get_mview_last_refresh_info(src_mview_ids, + sql_proxy, + tenant_id, + read_snapshot, + mview_ids, + mview_refresh_scns, + refresh_modes))) { + LOG_WARN("fail to get mview last refresh info", K(ret), K(src_mview_ids), K(tenant_id)); + } else if (OB_FAIL(ObMViewMaintenanceService:: + update_mview_refresh_info_cache(mview_ids, + mview_refresh_scns, + refresh_modes, + mview_refresh_info_map_))) { + LOG_WARN("fail to update mview refresh info cache", K(ret), K(tenant_id)); + } + } + // for debug + LOG_INFO("use mview refresh info cache", + K(src_mview_ids), K(mview_ids), + K(tenant_id), K(not_hit_count), + K(mview_refresh_scns), K(read_snapshot)); + return ret; +} } // namespace rootserver } // namespace oceanbase diff --git a/src/rootserver/mview/ob_mview_maintenance_service.h b/src/rootserver/mview/ob_mview_maintenance_service.h index c5a1ec864a..3e189bd94c 100644 --- a/src/rootserver/mview/ob_mview_maintenance_service.h +++ b/src/rootserver/mview/ob_mview_maintenance_service.h @@ -16,16 +16,41 @@ #include "rootserver/mview/ob_mlog_maintenance_task.h" #include "rootserver/mview/ob_mview_maintenance_task.h" #include "rootserver/mview/ob_mview_refresh_stats_maintenance_task.h" +#include "rootserver/mview/ob_mview_push_refresh_scn_task.h" +#include "rootserver/mview/ob_mview_push_snapshot_task.h" +#include "rootserver/mview/ob_collect_mv_merge_info_task.h" +#include "rootserver/mview/ob_mview_clean_snapshot_task.h" #include "share/scn.h" +#include "rootserver/mview/ob_replica_safe_check_task.h" +#include "rootserver/mview/ob_mview_update_cache_task.h" namespace oceanbase { namespace rootserver { +class ObMviewUpdateCacheTask; +struct RefreshInfo +{ + uint64_t refresh_scn_; + int64_t refresh_ts_; + int64_t expired_ts_; + bool hit_cache(const share::SCN &read_snapshot) { + return expired_ts_ >= read_snapshot.convert_to_ts() && + read_snapshot.get_val_for_tx() >= refresh_scn_; + } + RefreshInfo() : refresh_scn_(0), refresh_ts_(0), expired_ts_(0) + {}; +}; + +typedef common::hash:: + ObHashMap + ObMviewRefreshInfoMap; class ObMViewMaintenanceService : public logservice::ObIReplaySubHandler, public logservice::ObICheckpointSubHandler, public logservice::ObIRoleChangeSubHandler { +public: + static const int64_t CacheValidInterval = 30 * 1000 * 1000; //30s public: ObMViewMaintenanceService(); virtual ~ObMViewMaintenanceService(); @@ -64,6 +89,34 @@ public: int switch_to_follower_gracefully() override final; int resume_leader() override final; + int get_mview_refresh_info(const ObIArray &src_mview_ids, + ObMySQLProxy *sql_proxy, + const share::SCN &read_snapshot, + ObIArray &mview_ids, + ObIArray &mview_refresh_scns); + int get_mview_last_refresh_info(const ObIArray &src_mview_ids, + ObMySQLProxy *sql_proxy, + const uint64_t tenant_id, + const share::SCN &scn, + ObIArray &mview_ids, + ObIArray &last_refresh_scns, + ObIArray &mview_refresh_modes); + static int extract_sql_result(sqlclient::ObMySQLResult *mysql_result, + ObIArray &mview_ids, + ObIArray &last_refresh_scns, + ObIArray &mview_refresh_modes); + static int update_mview_refresh_info_cache(const ObIArray &mview_ids, + const ObIArray &mview_refresh_scns, + const ObIArray &mview_refresh_modes, + ObMviewRefreshInfoMap &mview_refresh_info_map); + int fetch_mv_refresh_scns(const ObIArray &src_mview_ids, + const share::SCN &read_snapshot, + ObIArray &mview_ids, + ObIArray &mview_refresh_scns, + uint64_t ¬_hit_count); + int64_t get_last_request_ts() const { return ATOMIC_LOAD(&last_request_ts_); } + void set_last_request_ts(const int64_t last_request_ts) { ATOMIC_STORE(&last_request_ts_, last_request_ts); } + ObMviewRefreshInfoMap &get_mview_refresh_info_map() { return mview_refresh_info_map_; } private: int inner_switch_to_leader(); int inner_switch_to_follower(); @@ -72,6 +125,14 @@ private: ObMLogMaintenanceTask mlog_maintenance_task_; ObMViewMaintenanceTask mview_maintenance_task_; ObMViewRefreshStatsMaintenanceTask mvref_stats_maintenance_task_; + ObMViewPushRefreshScnTask mview_push_refresh_scn_task_; + ObMViewPushSnapshotTask mview_push_snapshot_task_; + ObReplicaSafeCheckTask replica_safe_check_task_; + ObCollectMvMergeInfoTask collect_mv_merge_info_task_; + ObMViewCleanSnapshotTask mview_clean_snapshot_task_; + ObMviewUpdateCacheTask mview_update_cache_task_; + ObMviewRefreshInfoMap mview_refresh_info_map_; + int64_t last_request_ts_; bool is_inited_; }; diff --git a/src/rootserver/mview/ob_mview_maintenance_task.h b/src/rootserver/mview/ob_mview_maintenance_task.h index 2177243ec9..ccb4da4650 100644 --- a/src/rootserver/mview/ob_mview_maintenance_task.h +++ b/src/rootserver/mview/ob_mview_maintenance_task.h @@ -37,7 +37,7 @@ public: void runTimerTask() override; private: - static const int64_t MVIEW_MAINTENANCE_INTERVAL = 24LL * 3600 * 1000 * 1000; // 1day + static const int64_t MVIEW_MAINTENANCE_INTERVAL = 60 * 1000 * 1000; // 1min static const int64_t MVIEW_MAINTENANCE_SCHED_INTERVAL = 10LL * 1000 * 1000; // 10s static const int64_t MVIEW_NUM_FETCH_PER_SCHED = 1000; static const int64_t MVREF_STATS_NUM_PURGE_PER_SCHED = 1000; diff --git a/src/rootserver/mview/ob_mview_push_refresh_scn_task.cpp b/src/rootserver/mview/ob_mview_push_refresh_scn_task.cpp new file mode 100644 index 0000000000..d5a30a2242 --- /dev/null +++ b/src/rootserver/mview/ob_mview_push_refresh_scn_task.cpp @@ -0,0 +1,268 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX RS + +#include "observer/dbms_scheduler/ob_dbms_sched_job_utils.h" +#include "observer/ob_server_struct.h" +#include "observer/omt/ob_multi_tenant.h" +#include "rootserver/mview/ob_mview_push_refresh_scn_task.h" +#include "share/ob_errno.h" +#include "share/schema/ob_mview_info.h" +#include "share/schema/ob_mview_refresh_stats_params.h" +#include "share/ob_global_stat_proxy.h" +#include "share/ob_freeze_info_proxy.h" +#include "share/backup/ob_backup_data_table_operator.h" +#include "storage/mview/ob_mview_refresh_stats_purge.h" + +namespace oceanbase { +namespace rootserver { + +#define QUERY_MAJOR_MV_MERGE_SCN_SQL "select mview_id,t2.data_table_id,last_refresh_scn,t3.tablet_id, \ + t4.svr_ip,t4.svr_port,t4.end_log_scn from __all_mview t1 \ + left join __all_table t2 on t1.mview_id = t2.table_id \ + left join __all_tablet_to_ls t3 on t2.data_table_id = t3.table_id \ + left join __all_virtual_table_mgr t4 on t3.tablet_id = t4.tablet_id and t4.table_type = 10 \ + where t1.refresh_mode = 4 and t1.last_refresh_scn > 0 order by 1,2,3,4,5,6,7" + + +ObMViewPushRefreshScnTask::ObMViewPushRefreshScnTask() + : is_inited_(false), + in_sched_(false), + is_stop_(true), + tenant_id_(OB_INVALID_TENANT_ID) +{ +} + +ObMViewPushRefreshScnTask::~ObMViewPushRefreshScnTask() {} + +int ObMViewPushRefreshScnTask::init() +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObMViewPushRefreshScnTask init twice", KR(ret), KP(this)); + } else { + tenant_id_ = MTL_ID(); + is_inited_ = true; + } + return ret; +} + +int ObMViewPushRefreshScnTask::start() +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObMViewPushRefreshScnTask not init", KR(ret), KP(this)); + } else { + is_stop_ = false; + if (!in_sched_ && + OB_FAIL(schedule_task(MVIEW_PUSH_REFRESH_SCN_INTERVAL, true /*repeat*/))) { + LOG_WARN("fail to schedule ObMViewPushRefreshScnTask", KR(ret)); + } else { + in_sched_ = true; + } + } + return ret; +} + +void ObMViewPushRefreshScnTask::stop() +{ + is_stop_ = true; + in_sched_ = false; + cancel_task(); +} + +void ObMViewPushRefreshScnTask::destroy() +{ + is_inited_ = false; + is_stop_ = true; + in_sched_ = false; + cancel_task(); + wait_task(); + tenant_id_ = OB_INVALID_TENANT_ID; +} + +void ObMViewPushRefreshScnTask::wait() { wait_task(); } + +void ObMViewPushRefreshScnTask::runTimerTask() +{ + int ret = OB_SUCCESS; + uint64_t data_version = 0; + common::ObISQLClient *sql_proxy = GCTX.sql_proxy_; + bool need_schedule = false; + ObMySQLTransaction trans; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObMViewPushRefreshScnTask not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(is_stop_)) { + // do nothing + } else if (OB_FAIL(need_schedule_major_refresh_mv_task(tenant_id_, need_schedule))) { + LOG_WARN("fail to check need schedule major refresh mv task", KR(ret), K(tenant_id_)); + } else if (!need_schedule) { + } else if (FALSE_IT(void(check_major_mv_refresh_scn_safety(tenant_id_)))) { + } else if (OB_UNLIKELY(OB_ISNULL(sql_proxy))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else if (OB_FAIL(trans.start(sql_proxy, tenant_id_))) { + LOG_WARN("fail to start trans", KR(ret), K(tenant_id_)); + } else { + share::ObGlobalStatProxy stat_proxy(trans, tenant_id_); + share::SCN major_refresh_mv_merge_scn; + ObArray backup_jobs; + uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_); + if (OB_FAIL(stat_proxy.get_major_refresh_mv_merge_scn(true /*select for update*/, + major_refresh_mv_merge_scn))) { + LOG_WARN("fail to get major_refresh_mv_merge_scn", KR(ret), K(tenant_id_)); + } else if (OB_UNLIKELY(!major_refresh_mv_merge_scn.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("major_refresh_mv_merge_scn is invalid", KR(ret), K(tenant_id_), + K(major_refresh_mv_merge_scn)); + } else if (OB_FAIL(share::ObBackupJobOperator::get_jobs( + *sql_proxy, meta_tenant_id, false /*select for update*/, backup_jobs))) { + LOG_WARN("failed to get backup jobs", K(ret), K(tenant_id_)); + } else if (!backup_jobs.empty()) { + LOG_INFO("[MAJ_REF_MV] backup jobs exist, skip push major refresh mv scn", KR(ret), + K(tenant_id_)); + } else if (OB_FAIL(ObMViewInfo::update_major_refresh_mview_scn(trans, tenant_id_, + major_refresh_mv_merge_scn))) { + LOG_WARN("fail to update major_refresh_mview_scn", KR(ret), K(tenant_id_), + K(major_refresh_mv_merge_scn)); + } else { + LOG_INFO("[MAJ_REF_MV] successfully push major refresh mview refresh scn", K(tenant_id_), + K(major_refresh_mv_merge_scn)); + } + } + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + } +} + +int ObMViewPushRefreshScnTask::check_major_mv_refresh_scn_safety(const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + ObMySQLTransaction trans; + common::ObISQLClient *sql_proxy = GCTX.sql_proxy_; + ObArray merge_info_array; + if (OB_FAIL(trans.start(sql_proxy, tenant_id))) { + LOG_WARN("fail to start trans", KR(ret), K(tenant_id)); + } else { + share::ObGlobalStatProxy stat_proxy(trans, tenant_id); + share::SCN major_refresh_mv_merge_scn; + const bool select_for_update = true; + if (OB_FAIL(stat_proxy.get_major_refresh_mv_merge_scn(select_for_update, + major_refresh_mv_merge_scn))) { + LOG_WARN("fail to get major_refresh_mv_merge_scn", KR(ret), K(tenant_id)); + } else if (OB_FAIL(get_major_mv_merge_info_(tenant_id, trans, merge_info_array))) { + LOG_WARN("fail to get major_mv merge_info", KR(ret)); + } + } + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + } + // TODO ignore learn list + if (OB_SUCC(ret)) { + LOG_INFO("major_mv_safety>>>>"); + bool is_safety = true; + for (int64_t idx = 0; OB_SUCC(ret) && idx < merge_info_array.count(); idx++) { + ObMajorMVMergeInfo &merge_info = merge_info_array.at(idx); + LOG_INFO("major_mv_safety>>>> merge_info", K(merge_info)); + if (!merge_info.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("merge_info is invalid", K(merge_info)); + } else if (idx == merge_info_array.count() - 1 || !merge_info.is_equal_node(merge_info_array.at(idx+1))) { + bool find_dest_merge_scn = false; + for (int64_t i = 0; i < merge_info_array.count(); i++) { + ObMajorMVMergeInfo &tmp_merge_info = merge_info_array.at(i); + if (merge_info.is_equal_node(tmp_merge_info)) { + if (merge_info.last_refresh_scn_ == tmp_merge_info.end_log_scn_) { + find_dest_merge_scn = true; + break; + } + } + } + if (!find_dest_merge_scn) { + LOG_ERROR("major_mv_safety>>>>", K(merge_info)); + is_safety = false; + } + } + } + LOG_INFO("major_mv_safety>>>>", K(is_safety)); + } + return ret; +} + +int ObMViewPushRefreshScnTask::get_major_mv_merge_info_(const uint64_t tenant_id, + ObISQLClient &sql_client, + ObIArray &merge_info_array) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + if (OB_FAIL(sql.assign_fmt(QUERY_MAJOR_MV_MERGE_SCN_SQL))) { + LOG_WARN("assign sql failed", KR(ret)); + } else { + common::sqlclient::ObMySQLResult *result = nullptr; + SMART_VAR(ObMySQLProxy::MySQLResult, res) + { + common::sqlclient::ObMySQLResult *result = nullptr; + if (OB_FAIL(sql_client.read(res, tenant_id, sql.ptr()))) { + LOG_WARN("execute sql failed", KR(ret), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is null", KR(ret)); + } { + bool is_result_next_err = true; + while (OB_SUCC(ret) && OB_SUCC(result->next())) { + ObMajorMVMergeInfo merge_info; + char svr_ip[OB_IP_STR_BUFF] = ""; + int64_t svr_port = 0; + int64_t tmp_real_str_len = 0; + EXTRACT_INT_FIELD_MYSQL(*result, "mview_id", merge_info.mview_id_, int64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "data_table_id", merge_info.data_table_id_, int64_t); + EXTRACT_UINT_FIELD_MYSQL(*result, "last_refresh_scn", merge_info.last_refresh_scn_, uint64_t); + EXTRACT_INT_FIELD_MYSQL(*result, "tablet_id", merge_info.tablet_id_, int64_t); + EXTRACT_STRBUF_FIELD_MYSQL(*result, "svr_ip", svr_ip, OB_IP_STR_BUFF, tmp_real_str_len); + EXTRACT_INT_FIELD_MYSQL(*result, "svr_port", svr_port, int64_t); + (void)merge_info.svr_addr_.set_ip_addr(svr_ip, static_cast(svr_port)); + EXTRACT_UINT_FIELD_MYSQL(*result, "end_log_scn", merge_info.end_log_scn_, uint64_t); + + if (OB_FAIL(ret)) { + LOG_WARN("fail to extract field from result", KR(ret)); + } else if (OB_FAIL(merge_info_array.push_back(merge_info))) { + LOG_WARN("fail to push merge_info to array", KR(ret)); + } + if (OB_FAIL(ret)) { + is_result_next_err = false; + } + } + if (OB_ITER_END == ret && is_result_next_err) { + ret = OB_SUCCESS; + } + } + } + } + return ret; +} + +} // namespace rootserver +} // namespace oceanbase diff --git a/src/rootserver/mview/ob_mview_push_refresh_scn_task.h b/src/rootserver/mview/ob_mview_push_refresh_scn_task.h new file mode 100644 index 0000000000..0ebb30a1a6 --- /dev/null +++ b/src/rootserver/mview/ob_mview_push_refresh_scn_task.h @@ -0,0 +1,90 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#pragma once + +#include "lib/task/ob_timer.h" +#include "rootserver/mview/ob_mview_timer_task.h" +#include "lib/container/ob_iarray.h" + +namespace oceanbase +{ +namespace common +{ +class ObISQLClient; +} + +namespace rootserver +{ +// just for check major mv safety +struct ObMajorMVMergeInfo +{ +public: + ObMajorMVMergeInfo() : mview_id_(0), + data_table_id_(0), + last_refresh_scn_(0), + tablet_id_(0), + svr_addr_(), + end_log_scn_(0) + {} + ~ObMajorMVMergeInfo() {} + TO_STRING_KV(K_(mview_id), K_(data_table_id), K_(last_refresh_scn), K_(tablet_id), + K_(svr_addr), K_(end_log_scn)); + bool is_valid() { + return mview_id_ > 0 && data_table_id_ > 0 && last_refresh_scn_ > 0 + && svr_addr_.is_valid() && tablet_id_ > 0; + } + bool is_equal_node(ObMajorMVMergeInfo &other) { + return mview_id_ == other.mview_id_ && data_table_id_ == other.data_table_id_ + && last_refresh_scn_ == other.last_refresh_scn_ && tablet_id_ == other.tablet_id_ + && svr_addr_ == other.svr_addr_; + } +public: + int64_t mview_id_; + int64_t data_table_id_; + uint64_t last_refresh_scn_; + int64_t tablet_id_; + ObAddr svr_addr_; + uint64_t end_log_scn_; +}; + +class ObMViewPushRefreshScnTask : public ObMViewTimerTask +{ +public: + ObMViewPushRefreshScnTask(); + virtual ~ObMViewPushRefreshScnTask(); + DISABLE_COPY_ASSIGN(ObMViewPushRefreshScnTask); + // for Service + int init(); + int start(); + void stop(); + void wait(); + void destroy(); + // for TimerTask + void runTimerTask() override; + + static int check_major_mv_refresh_scn_safety(const uint64_t tenant_id); + static const int64_t MVIEW_PUSH_REFRESH_SCN_INTERVAL = 30 * 1000 * 1000; // 30s +private: + static int get_major_mv_merge_info_(const uint64_t tenant_id, + ObISQLClient &sql_client, + ObIArray &merge_info_array); +private: + bool is_inited_; + bool in_sched_; + bool is_stop_; + uint64_t tenant_id_; +}; + + +} // namespace rootserver +} // namespace oceanbase diff --git a/src/rootserver/mview/ob_mview_push_snapshot_task.cpp b/src/rootserver/mview/ob_mview_push_snapshot_task.cpp new file mode 100644 index 0000000000..76a93d24eb --- /dev/null +++ b/src/rootserver/mview/ob_mview_push_snapshot_task.cpp @@ -0,0 +1,158 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX RS + +#include "observer/dbms_scheduler/ob_dbms_sched_job_utils.h" +#include "observer/ob_server_struct.h" +#include "observer/omt/ob_multi_tenant.h" +#include "rootserver/mview/ob_mview_push_snapshot_task.h" +#include "share/ob_errno.h" +#include "share/schema/ob_mview_info.h" +#include "share/schema/ob_mview_refresh_stats_params.h" +#include "share/ob_global_stat_proxy.h" +#include "share/ob_snapshot_table_proxy.h" +#include "storage/mview/ob_mview_refresh_stats_purge.h" +#include "storage/compaction/ob_tenant_freeze_info_mgr.h" + +namespace oceanbase { +namespace rootserver { + +ObMViewPushSnapshotTask::ObMViewPushSnapshotTask() + : is_inited_(false), + in_sched_(false), + is_stop_(true), + tenant_id_(OB_INVALID_TENANT_ID) +{ +} + +ObMViewPushSnapshotTask::~ObMViewPushSnapshotTask() {} + +int ObMViewPushSnapshotTask::init() +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObMViewPushSnapshotTask init twice", KR(ret), KP(this)); + } else { + tenant_id_ = MTL_ID(); + is_inited_ = true; + } + return ret; +} + +int ObMViewPushSnapshotTask::start() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObMViewPushSnapshotTask not init", KR(ret), KP(this)); + } else { + is_stop_ = false; + if (!in_sched_ && OB_FAIL(schedule_task(MVIEW_PUSH_SNAPSHOT_INTERVAL, true /*repeat*/))) { + LOG_WARN("fail to schedule ObMViewPushSnapshotTask", KR(ret)); + } else { + in_sched_ = true; + } + } + return ret; +} + +void ObMViewPushSnapshotTask::stop() +{ + is_stop_ = true; + in_sched_ = false; + cancel_task(); +} + +void ObMViewPushSnapshotTask::destroy() +{ + is_inited_ = false; + is_stop_ = true; + in_sched_ = false; + cancel_task(); + wait_task(); + tenant_id_ = OB_INVALID_TENANT_ID; +} + +void ObMViewPushSnapshotTask::wait() { wait_task(); } + +void ObMViewPushSnapshotTask::runTimerTask() +{ + int ret = OB_SUCCESS; + uint64_t data_version = 0; + common::ObISQLClient *sql_proxy = GCTX.sql_proxy_; + storage::ObTenantFreezeInfoMgr *mgr = MTL(storage::ObTenantFreezeInfoMgr *); + bool need_schedule = false; + ObMySQLTransaction trans; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObMViewPushSnapshotTask not init", KR(ret), KP(this)); + } else if (OB_UNLIKELY(is_stop_)) { + + } else if (OB_FAIL(need_schedule_major_refresh_mv_task(tenant_id_, need_schedule))) { + LOG_WARN("fail to check need schedule major refresh mv task", KR(ret), K(tenant_id_)); + } else if (!need_schedule) { + + } else if (OB_UNLIKELY(OB_ISNULL(sql_proxy) || OB_ISNULL(mgr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql_proxy or mgr is null", KR(ret), K(sql_proxy), K(mgr)); + } else if (OB_FAIL(trans.start(sql_proxy, tenant_id_))) { + LOG_WARN("fail to start trans", KR(ret), K(tenant_id_)); + } else { + share::ObGlobalStatProxy stat_proxy(trans, tenant_id_); + share::SCN major_refresh_mv_merge_scn; + const int64_t snapshot_for_tx = mgr->get_min_reserved_snapshot_for_tx(); + share::SCN min_refresh_scn; + share::ObSnapshotTableProxy snapshot_proxy; + const bool select_for_update = true; + // we query the major_refresh_mv_merge_scn in __all_core_table to conflict with the backup + // process. + if (OB_FAIL(stat_proxy.get_major_refresh_mv_merge_scn(select_for_update, + major_refresh_mv_merge_scn))) { + LOG_WARN("fail to get major_refresh_mv_merge_scn", KR(ret), K(tenant_id_)); + } else if (OB_UNLIKELY(!major_refresh_mv_merge_scn.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("major_refresh_mv_merge_scn is invalid", KR(ret), K(tenant_id_), + K(major_refresh_mv_merge_scn)); + } // to ensure the concurrent query won't return the OB_SNAPSHOT_DISCARDED error, + // we use snapshot_for_tx to get the min refresh scn in __all_mview. + else if (OB_FAIL(ObMViewInfo::get_min_major_refresh_mview_scn( + trans, tenant_id_, snapshot_for_tx, min_refresh_scn))) { + // the tenant has no major refresh mview + if (OB_ERR_NULL_VALUE == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to get min major_refresh_mview_scn", KR(ret), K(tenant_id_), + K(snapshot_for_tx)); + } + } else if (OB_FAIL(snapshot_proxy.push_snapshot_for_major_refresh_mv(trans, tenant_id_, + min_refresh_scn))) { + LOG_WARN("fail to push snapshot for major refresh mv", KR(ret), K(tenant_id_), + K(min_refresh_scn)); + } else { + LOG_INFO("[MAJ_REF_MV] successfully push major refresh mview snapshot", K(tenant_id_), + K(snapshot_for_tx), K(min_refresh_scn)); + } + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + } + } +} + +} // namespace rootserver +} // namespace oceanbase diff --git a/src/rootserver/mview/ob_mview_push_snapshot_task.h b/src/rootserver/mview/ob_mview_push_snapshot_task.h new file mode 100644 index 0000000000..4b301bb51d --- /dev/null +++ b/src/rootserver/mview/ob_mview_push_snapshot_task.h @@ -0,0 +1,45 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#pragma once + +#include "lib/task/ob_timer.h" +#include "rootserver/mview/ob_mview_timer_task.h" + +namespace oceanbase +{ +namespace rootserver +{ +class ObMViewPushSnapshotTask : public ObMViewTimerTask +{ +public: + ObMViewPushSnapshotTask(); + virtual ~ObMViewPushSnapshotTask(); + DISABLE_COPY_ASSIGN(ObMViewPushSnapshotTask); + // for Service + int init(); + int start(); + void stop(); + void wait(); + void destroy(); + // for TimerTask + void runTimerTask() override; + static const int64_t MVIEW_PUSH_SNAPSHOT_INTERVAL = 60 * 1000 * 1000; // 1min +private: + bool is_inited_; + bool in_sched_; + bool is_stop_; + uint64_t tenant_id_; +}; + +} // namespace rootserver +} // namespace oceanbase diff --git a/src/rootserver/mview/ob_mview_timer_task.cpp b/src/rootserver/mview/ob_mview_timer_task.cpp index b3057ab622..9a0bef3b3b 100644 --- a/src/rootserver/mview/ob_mview_timer_task.cpp +++ b/src/rootserver/mview/ob_mview_timer_task.cpp @@ -15,6 +15,11 @@ #include "rootserver/mview/ob_mview_timer_task.h" #include "observer/omt/ob_multi_tenant.h" #include "share/ob_errno.h" +#include "share/ob_snapshot_table_proxy.h" +#include "share/schema/ob_mview_info.h" +#include "storage/compaction/ob_tenant_tablet_scheduler.h" +#include "share/ob_global_stat_proxy.h" +#include "share/schema/ob_schema_struct.h" namespace oceanbase { @@ -60,5 +65,141 @@ void ObMViewTimerTask::wait_task() } } +int ObMViewTimerTask::need_schedule_major_refresh_mv_task(const uint64_t tenant_id, + bool &need_schedule) +{ + int ret = OB_SUCCESS; + uint64_t data_version = 0; + common::ObISQLClient *sql_proxy = GCTX.sql_proxy_; + bool contains_major_refresh_mview = false; + share::ObSnapshotTableProxy snapshot_proxy; + need_schedule = false; + + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { + LOG_WARN("fail to get data version", KR(ret), K(tenant_id)); + } else if (OB_UNLIKELY(data_version < DATA_VERSION_4_3_4_0)) { + } else if (tenant_id == OB_SYS_TENANT_ID) { + // skip sys tenant + } else if (OB_UNLIKELY(OB_ISNULL(sql_proxy))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else if (OB_FAIL(snapshot_proxy.check_snapshot_exist(*sql_proxy, tenant_id, + share::SNAPSHOT_FOR_MAJOR_REFRESH_MV, + contains_major_refresh_mview))) { + LOG_WARN("fail to check if tenant contains major refresh snapshot", KR(ret), K(tenant_id)); + } else if (contains_major_refresh_mview) { + need_schedule = true; + } + + return ret; +} + +int ObMViewTimerTask::need_push_major_mv_merge_scn(const uint64_t tenant_id, + bool &need_push, + share::SCN &lastest_merge_scn, + share::SCN &major_mv_merge_scn) +{ + int ret = OB_SUCCESS; + need_push = false; + bool need_schedule = false; + ObGlobalStatProxy global_proxy(*GCTX.sql_proxy_, tenant_id); + compaction::ObTenantTabletScheduler* tablet_scheduler = MTL(compaction::ObTenantTabletScheduler*); + + if (OB_FAIL(need_schedule_major_refresh_mv_task(tenant_id, need_schedule))) { + LOG_WARN("failed to check need schedule", KR(ret), K(tenant_id)); + } else if (!need_schedule) { + // do nothing + } else if (OB_ISNULL(tablet_scheduler)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("obj is null", KR(ret), KP(tablet_scheduler)); + } else if (OB_FAIL(lastest_merge_scn.convert_for_gts(tablet_scheduler->get_inner_table_merged_scn()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to convert_for_gts", KR(ret)); + } else if (OB_FAIL(global_proxy.get_major_refresh_mv_merge_scn(false /*select for update*/, + major_mv_merge_scn))) { + LOG_WARN("fail to get major_refresh_mv_merge_scn", KR(ret), K(tenant_id)); + } + + if (OB_FAIL(ret)) { + } else if (!need_schedule) { + } else if (!lastest_merge_scn.is_valid() || !major_mv_merge_scn.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("merge_scn is not valid", KR(ret), K(lastest_merge_scn), K(major_mv_merge_scn)); + } else if (lastest_merge_scn < major_mv_merge_scn) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("lastest merge_scn less than major_mv_merge_scn", KR(ret), K(lastest_merge_scn), K(major_mv_merge_scn)); + } else if (lastest_merge_scn == major_mv_merge_scn) { + } else { + need_push = true; + } + return ret; +} + +int ObMViewTimerTask::check_mview_last_refresh_scn(const uint64_t tenant_id, + const share::SCN &tenant_mv_merge_scn) +{ + int ret = OB_SUCCESS; + share::SCN last_refresh_scn = share::SCN::min_scn(); + ObSqlString sql; + const int64_t refresh_mode = (int64_t)ObMVRefreshMode::MAJOR_COMPACTION; // new mview mode + const int64_t last_refresh_type = (int64_t)ObMVRefreshType::FAST; // inc refresh type + const uint64_t user_tenant_id = gen_user_tenant_id(tenant_id); + if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql_proxy should not null", KR(ret), KP(GCTX.sql_proxy_)); + } else if (!is_valid_tenant_id(user_tenant_id) || !is_valid_tenant_id(tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant_id is invalid", KR(ret), K(user_tenant_id), K(tenant_id)); + } else if (OB_FAIL(sql.assign_fmt("SELECT MIN(LAST_REFRESH_SCN) AS MIN_LAST_REFRESH_SCN FROM \ + `%s`.`%s` \ + WHERE (REFRESH_MODE = %ld) AND (LAST_REFRESH_TYPE = %ld)", + OB_SYS_DATABASE_NAME, OB_ALL_MVIEW_TNAME, + refresh_mode, last_refresh_type))) { + LOG_WARN("fail to format sql", KR(ret), K(sql), K(user_tenant_id)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + sqlclient::ObMySQLResult *mysql_result = NULL; + if (OB_FAIL(GCTX.sql_proxy_->read(res, + user_tenant_id, + sql.ptr()))) { + LOG_WARN("fail to execute sql", K(ret), K(sql), K(user_tenant_id)); + } else if (OB_ISNULL(mysql_result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is null", K(ret), KP(mysql_result)); + } else if (OB_FAIL(mysql_result->next())) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to get next", K(ret), KP(mysql_result)); + } + } else { + uint64_t min_last_refresh_scn = 0; + EXTRACT_UINT_FIELD_MYSQL_WITH_DEFAULT_VALUE( + *mysql_result, "MIN_LAST_REFRESH_SCN", min_last_refresh_scn, uint64_t, + true/*skip null*/, false /*skip column*/, 0/*default value*/); + if (OB_SUCC(ret)) { + if (0 == min_last_refresh_scn) { + last_refresh_scn.set_min(); + } else { + last_refresh_scn.convert_from_ts(min_last_refresh_scn); + } + } + } + } + if (OB_FAIL(ret)) { + } else if (last_refresh_scn.is_min()) { + // major mv merge scn is zero or one + } else if (last_refresh_scn == tenant_mv_merge_scn) { + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("mv min last_refresh_scn not equal to major_mv_merge_scn", + K(ret), K(tenant_id), K(tenant_mv_merge_scn), K(last_refresh_scn)); + } + LOG_INFO("get mview last refresh scn", KR(ret), K(sql), K(user_tenant_id), + K(tenant_mv_merge_scn), K(last_refresh_scn)); + } + return ret; +} + } // namespace rootserver } // namespace oceanbase diff --git a/src/rootserver/mview/ob_mview_timer_task.h b/src/rootserver/mview/ob_mview_timer_task.h index dbaeff4858..073c4cc143 100644 --- a/src/rootserver/mview/ob_mview_timer_task.h +++ b/src/rootserver/mview/ob_mview_timer_task.h @@ -13,6 +13,7 @@ #pragma once #include "lib/task/ob_timer.h" +#include "share/scn.h" namespace oceanbase { @@ -27,6 +28,15 @@ public: int schedule_task(const int64_t delay, bool repeate = false, bool immediate = false); void cancel_task(); void wait_task(); + + static int need_schedule_major_refresh_mv_task(const uint64_t tenant_id, + bool &need_schedule); + static int need_push_major_mv_merge_scn(const uint64_t tenant_id, + bool &need_push, + share::SCN &latest_merge_scn, + share::SCN &major_mv_merge_scn); + static int check_mview_last_refresh_scn(const uint64_t tenant_id, + const share::SCN &tenant_mv_merge_scn); }; } // namespace rootserver diff --git a/src/rootserver/mview/ob_mview_update_cache_task.cpp b/src/rootserver/mview/ob_mview_update_cache_task.cpp new file mode 100644 index 0000000000..0a9b8da0da --- /dev/null +++ b/src/rootserver/mview/ob_mview_update_cache_task.cpp @@ -0,0 +1,170 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX RS +#include "rootserver/mview/ob_mview_update_cache_task.h" +#include "rootserver/mview/ob_mview_maintenance_service.h" +#include "share/schema/ob_schema_struct.h" // ObMvRefreshMode + +namespace oceanbase +{ +namespace rootserver +{ + +ObMviewUpdateCacheTask::ObMviewUpdateCacheTask() + : is_inited_(false), + is_stop_(false), + in_sched_(false) +{ +} +ObMviewUpdateCacheTask::~ObMviewUpdateCacheTask() +{ + clean_up(); +} + +int ObMviewUpdateCacheTask::init() +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObMviewUpdateCacheTask init twice", KR(ret), KPC(this)); + } else { + is_inited_ = true; + } + return ret; +} + +int ObMviewUpdateCacheTask::start() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObMviewUpdateCacheTask not init", KR(ret), KPC(this)); + } else { + is_stop_ = false; + if (!in_sched_ && OB_FAIL(schedule_task(TaskDelay, true /*repeat*/))) { + LOG_WARN("fail to schedule update mview cache task", KR(ret)); + } else { + in_sched_ = true; + LOG_INFO("ObMviewUpdateCacheTask started", KR(ret), KPC(this)); + } + } + return ret; +} + +void ObMviewUpdateCacheTask::stop() +{ + is_stop_ = true; + in_sched_ = false; + cancel_task(); +} + +void ObMviewUpdateCacheTask::wait() { wait_task(); } +void ObMviewUpdateCacheTask::destroy() +{ + is_inited_ = false; + is_stop_ = true; + in_sched_ = false; + cancel_task(); + wait_task(); + clean_up(); +} + +void ObMviewUpdateCacheTask::clean_up() +{ + is_inited_ = false; + is_stop_ = false; + in_sched_ = false; +} + +int ObMviewUpdateCacheTask::get_mview_refresh_scn_sql(const int refresh_mode, + ObSqlString &sql) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(sql.assign_fmt("SELECT CAST(MVIEW_ID AS UNSIGNED) AS MVIEW_ID, \ + LAST_REFRESH_SCN, \ + CAST(REFRESH_MODE AS UNSIGNED) AS REFRESH_MODE \ + FROM `%s`.`%s` \ + WHERE TENANT_ID = 0 and REFRESH_MODE = %d", + OB_SYS_DATABASE_NAME, OB_ALL_MVIEW_TNAME, refresh_mode))) { + LOG_WARN("fail to get sql", KR(ret), K(sql)); + } + return ret; +} + +void ObMviewUpdateCacheTask::runTimerTask() +{ + int ret = OB_SUCCESS; + ObSqlString sql; + share::SCN read_snapshot(share::SCN::min_scn()); + share::SCN mview_refresh_snapshot; + const uint64_t tenant_id = MTL_ID(); + ObMySQLProxy *sql_proxy = GCTX.sql_proxy_; + rootserver::ObMViewMaintenanceService *mview_maintenance_service = + MTL(rootserver::ObMViewMaintenanceService*); + int64_t current_ts = ObTimeUtility::fast_current_time(); + int64_t last_request_ts; + const int64_t NeedUpdateCacheInterval = 10 * 60 * 1000 * 1000; // 10min + // check request time; + if (OB_ISNULL(sql_proxy) || OB_ISNULL(mview_maintenance_service)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null or ObMViewMaintenanceService is null", + KR(ret), KP(sql_proxy), KP(mview_maintenance_service)); + } else if (!is_valid_tenant_id(tenant_id)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant id is invalid", KR(ret), K(tenant_id)); + } else if (OB_FALSE_IT(last_request_ts = mview_maintenance_service->get_last_request_ts())) { + } else if (last_request_ts < current_ts && + current_ts - last_request_ts > NeedUpdateCacheInterval) { + // clear cache + if (!mview_maintenance_service->get_mview_refresh_info_map().empty()) { + mview_maintenance_service->get_mview_refresh_info_map().clear(); + } + } else { + const int64_t refresh_mode = (int64_t)ObMVRefreshMode::MAJOR_COMPACTION; + ObMviewRefreshInfoMap &mview_refresh_info_map = mview_maintenance_service->get_mview_refresh_info_map(); + ObSEArray mview_ids; + ObSEArray mview_refresh_scns; + ObSEArray mview_refresh_modes; + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + ObSqlString sql; + sqlclient::ObMySQLResult *mysql_result = NULL; + if (OB_FAIL(get_mview_refresh_scn_sql(refresh_mode, sql))) { + LOG_WARN("failed to get last refresh scn sql", K(ret)); + } else if (OB_FAIL(sql_proxy->read(res, + tenant_id, + sql.ptr()))) { + LOG_WARN("fail to execute sql", K(ret), K(sql), K(tenant_id)); + } else if (OB_FAIL(ObMViewMaintenanceService:: + extract_sql_result(res.get_result(), + mview_ids, + mview_refresh_scns, + mview_refresh_modes))) { + LOG_WARN("fail to extract sql result", K(ret), K(sql), K(tenant_id)); + } + } + if (OB_FAIL(ret)) { + //do nothing + } else if (mview_ids.empty()) { + // do nothing + } else if (OB_FAIL(ObMViewMaintenanceService:: + update_mview_refresh_info_cache(mview_ids, + mview_refresh_scns, + mview_refresh_modes, + mview_refresh_info_map))){ + LOG_WARN("fail to update mview refresh info cache", K(ret), + K(mview_ids), K(mview_refresh_scns), K(mview_refresh_modes), K(tenant_id)); + } + } +} + +} +} \ No newline at end of file diff --git a/src/rootserver/mview/ob_mview_update_cache_task.h b/src/rootserver/mview/ob_mview_update_cache_task.h new file mode 100644 index 0000000000..d364f2a69d --- /dev/null +++ b/src/rootserver/mview/ob_mview_update_cache_task.h @@ -0,0 +1,54 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#pragma once + +#include "lib/task/ob_timer.h" +#include "rootserver/mview/ob_mview_timer_task.h" +#include "lib/container/ob_iarray.h" +#include "lib/hash/ob_hashmap.h" //ObHashMap +#include "src/sql/session/ob_sql_session_info.h" // ObSQLSessionInfo +#include "src/sql/engine/expr/ob_expr_last_refresh_scn.h" +#include "lib/atomic/ob_atomic.h" + +namespace oceanbase +{ + +namespace rootserver +{ +class ObMViewMaintenanceService; + +class ObMviewUpdateCacheTask : public ObMViewTimerTask +{ +public: + static const uint64_t TaskDelay = 5 * 1000 * 1000; // 5s +public: + ObMviewUpdateCacheTask(); + virtual ~ObMviewUpdateCacheTask(); + int get_mview_refresh_scn_sql(const int refresh_mode, ObSqlString &sql); + int init(); + int start(); + void stop(); + void wait(); + void destroy(); + void clean_up(); + void runTimerTask() override; + DISABLE_COPY_ASSIGN(ObMviewUpdateCacheTask); +private: + bool is_inited_; + bool is_stop_; + bool in_sched_; +}; + + +} // namespace rootserver +} // namespace oceanbase diff --git a/src/rootserver/mview/ob_replica_safe_check_task.cpp b/src/rootserver/mview/ob_replica_safe_check_task.cpp new file mode 100644 index 0000000000..8ccbcafc66 --- /dev/null +++ b/src/rootserver/mview/ob_replica_safe_check_task.cpp @@ -0,0 +1,475 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE + +#include "share/ob_errno.h" +#include "ob_replica_safe_check_task.h" +#include "observer/ob_server_struct.h" +#include "storage/compaction/ob_tenant_tablet_scheduler.h" +#include "share/ls/ob_ls_status_operator.h" +#include "share/ob_global_stat_proxy.h" + +namespace oceanbase +{ +namespace rootserver +{ + +int ObMVMergeSCNInfoCache::get_ls_info( + const share::ObLSID &ls_id, + ObMVMergeSCNInfo *&ls_info) +{ + int ret = OB_SUCCESS; + ls_info = NULL; + if (!ls_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("arg invalid", KR(ret), K(ls_id)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < arr_.count(); i++) { + ObMVMergeSCNInfo &info = arr_.at(i); + if (!info.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("info is invalid", KR(ret), K(ls_id), K(info)); + } else if (info.ls_id_ == ls_id) { + ls_info = &info; + break; + } + } + + if (OB_SUCC(ret) && OB_ISNULL(ls_info)) { + if (OB_FAIL(arr_.push_back(ObMVMergeSCNInfo(ls_id)))) { + LOG_WARN("failed to push_back", KR(ret), K(ls_id)); + } else { + ls_info = &arr_.at(arr_.count() - 1); + LOG_INFO("add ls_info", KR(ret), K(ls_id), KPC(ls_info), KPC(this)); + } + } + } + return ret; +} + +int ObMVMergeSCNInfoCache::clear_deleted_ls_info( + const share::SCN &merge_scn) +{ + int ret = OB_SUCCESS; + if (!merge_scn.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("arg is invalid", KR(ret), K(merge_scn), KPC(this)); + } else { + for (int64_t i = arr_.count() - 1; OB_SUCC(ret) && i >= 0; i--) { + ObMVMergeSCNInfo &info = arr_.at(i); + if (!info.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("info is invalid", KR(ret), K(info), KPC(this)); + } else if (info.major_mv_merge_scn_publish_ > merge_scn) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("info.major_mv_merge_scn_publish_ is more than merge_scn", KR(ret), K(merge_scn), K(info), KPC(this)); + } else if (info.major_mv_merge_scn_publish_ < merge_scn) { + arr_.remove(i); + LOG_INFO("delete ls info", K(i), K(info), K(merge_scn), KPC(this)); + } + } + } + return ret; +} + +ObReplicaSafeCheckTask::ObReplicaSafeCheckTask() + : status_(StatusType::PUBLISH_SCN), + in_sched_(false), + is_stop_(true), + is_inited_(false), + merge_scn_(), + max_transfer_task_id_() +{ +} + +ObReplicaSafeCheckTask::~ObReplicaSafeCheckTask() {} + +int ObReplicaSafeCheckTask::init() +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObReplicaSafeCheckTask init twice", KR(ret), KPC(this)); + } else { + is_inited_ = true; + } + return ret; +} + +int ObReplicaSafeCheckTask::start() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObReplicaSafeCheckTask not init", KR(ret), KPC(this)); + } else { + is_stop_ = false; + if (!in_sched_ && OB_FAIL(schedule_task(CHECK_INTERVAL, false /*repeat*/))) { + LOG_WARN("fail to schedule mlog maintenance task", KR(ret)); + } else { + in_sched_ = true; + LOG_INFO("ObReplicaSafeCheckTask started", KR(ret), KPC(this)); + } + } + return ret; +} + +void ObReplicaSafeCheckTask::stop() +{ + is_stop_ = true; + in_sched_ = false; + cancel_task(); + LOG_INFO("ObReplicaSafeCheckTask stopped", KPC(this)); +} + +void ObReplicaSafeCheckTask::wait() { wait_task(); } +void ObReplicaSafeCheckTask::destroy() +{ + is_inited_ = false; + is_stop_ = true; + in_sched_ = false; + cancel_task(); + wait_task(); + cleanup(); +} + +void ObReplicaSafeCheckTask::runTimerTask() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObReplicaSafeCheckTask not init", KR(ret), KPC(this)); + } else if (OB_UNLIKELY(is_stop_)) { + // do nothing + } else { + switch (status_) { + case StatusType::PUBLISH_SCN: + if (OB_FAIL(publish_scn())) { + LOG_WARN("fail to publish scn", KR(ret), KPC(this)); + } + break; + case StatusType::CHECK_END: + if (OB_FAIL(check_end())) { + LOG_WARN("fail to check end", KR(ret), KPC(this)); + } + break; + case StatusType::NOTICE_SAFE: + if (OB_FAIL(notice_safe())) { + LOG_WARN("fail to notice safe", KR(ret), KPC(this)); + } + break; + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected status", KR(ret), KPC(this)); + break; + } + } +} + +void ObReplicaSafeCheckTask::switch_status(StatusType new_status, int64_t delay) +{ + int ret = OB_SUCCESS; + // row: old_status; col: new_status + bool check_status[3][3] = { + {1, 1, 0}, + {1, 1, 1}, + {1, 0, 1} + }; + if (check_status[(int)status_][(int)new_status]) { + status_ = new_status; + if (in_sched_) { + if (OB_FAIL(schedule_task(delay, false /*repeat*/))) { + LOG_WARN("fail to schedule replica safe check task", KR(ret), KPC(this)); + } + } + LOG_INFO("replica safe check task switch_status", KR(ret), K(new_status), K(delay), KPC(this)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("status error", KR(ret), K(new_status), K(delay), KPC(this)); + ob_abort(); + } +} + +int ObReplicaSafeCheckTask::do_multi_trans( + const transaction::ObTxDataSourceType type, + const ObUpdateMergeScnArg &arg) +{ + int ret = OB_SUCCESS; + ObMySQLTransaction trans; + observer::ObInnerSQLConnection *conn = NULL; + int MAX_MULTI_BUF_SIZE = 64; + char buf[MAX_MULTI_BUF_SIZE]; + int64_t pos = 0; + int64_t buf_len = arg.get_serialize_size(); + if (share::SYS_LS == arg.ls_id_ || !arg.is_valid() + || (transaction::ObTxDataSourceType::MV_PUBLISH_SCN != type + && transaction::ObTxDataSourceType::MV_NOTICE_SAFE != type)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("arg is invaild", KR(ret), K(type), K(arg)); + } else if (buf_len > MAX_MULTI_BUF_SIZE) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("status error", KR(ret), K(buf_len), K(MAX_MULTI_BUF_SIZE), KPC(this)); + ob_abort(); + } else if (OB_FAIL(trans.start(GCTX.sql_proxy_, MTL_ID()))) { + LOG_WARN("failed to start trans", KR(ret), KPC(this)); + } else if (OB_ISNULL(conn = dynamic_cast + (trans.get_connection()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("conn is NULL", KR(ret), KPC(this)); + } else if (OB_FAIL(arg.serialize(buf, buf_len, pos))) { + LOG_WARN("fail to serialize", KR(ret), K(arg)); + } else if (OB_FAIL(conn->register_multi_data_source(MTL_ID(), arg.ls_id_, type, buf, buf_len))) { + LOG_WARN("fail to register_tx_data", KR(ret), K(arg), K(buf_len)); + } + if (trans.is_started()) { + int temp_ret = OB_SUCCESS; + if (OB_SUCCESS != (temp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("trans end failed", "is_commit", OB_SUCCESS == ret, K(temp_ret)); + ret = (OB_SUCC(ret)) ? temp_ret : ret; + } + } + return ret; +} + +int ObReplicaSafeCheckTask::publish_scn() +{ + int ret = OB_SUCCESS; + share::ObLSAttrArray ls_attr_array; + share::ObLSAttrOperator ls_operator(MTL_ID(), GCTX.sql_proxy_); + bool need_publish = true; + share::SCN lastest_merge_scn; + share::SCN major_mv_merge_scn; + uint64_t tenant_id = MTL_ID(); + if (OB_FAIL(need_push_major_mv_merge_scn(tenant_id, need_publish, lastest_merge_scn, major_mv_merge_scn))) { + LOG_WARN("fail to check need schedule major refresh mv task", KR(ret), K(MTL_ID())); + } else if (!need_publish) { + } else if (FALSE_IT(merge_scn_.atomic_store(lastest_merge_scn))) { + } else if (OB_FAIL(ls_operator.get_all_ls_by_order(ls_attr_array))) { + LOG_WARN("failed to get all ls status", KR(ret), KPC(this)); + } else { + int64_t publish_cnt = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < ls_attr_array.count(); ++i) { + const share::ObLSAttr &ls_attr = ls_attr_array.at(i); + const share::ObLSID &ls_id = ls_attr.get_ls_id(); + if (share::SYS_LS != ls_id && ls_attr.ls_is_normal()) { + ObMVMergeSCNInfo *ls_info = NULL; + if (OB_FAIL(ls_cache_.get_ls_info(ls_id, ls_info))) { + LOG_WARN("failed to get_ls_info", KR(ret), K(ls_id), KPC(this)); + } else if (OB_ISNULL(ls_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls_info is invalid", KR(ret), K(ls_id), K(ls_info), KPC(this)); + } else if (ls_info->major_mv_merge_scn_publish_ > merge_scn_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls_info.major_mv_merge_scn_publish_ is more than merge_scn", KR(ret), K(ls_id), K(ls_info), KPC(this)); + } else if (ls_info->major_mv_merge_scn_publish_ < merge_scn_) { + ObUpdateMergeScnArg arg; + arg.merge_scn_ = merge_scn_; + arg.ls_id_ = ls_id; + if (OB_FAIL(do_multi_trans(transaction::ObTxDataSourceType::MV_PUBLISH_SCN, arg))) { + LOG_WARN("failed to do multi trans", KR(ret), K(ls_attr), K(arg), K(this)); + } else { + publish_cnt++; + ls_info->major_mv_merge_scn_publish_ = merge_scn_; + } + } + } + } + if (OB_SUCC(ret)) { + if (0 == publish_cnt) { + need_publish = false; + } else if (OB_FAIL(ls_cache_.clear_deleted_ls_info(merge_scn_))) { + LOG_WARN("failed to clear_deleted_ls_info", KR(ret), K(publish_cnt), K(this)); + } + } + } + LOG_INFO("publish_scn finish", KR(ret), K(need_publish), K(major_mv_merge_scn), K(lastest_merge_scn), KPC(this), K(ls_attr_array)); + + if (OB_LS_LOCATION_LEADER_NOT_EXIST == ret || OB_NOT_MASTER == ret) { + switch_status(StatusType::PUBLISH_SCN, LOCATION_RETRY_INTERVAL); + } else if (OB_FAIL(ret)) { + switch_status(StatusType::PUBLISH_SCN, ERROR_RETRY_INTERVAL); + } else if (!need_publish) { + switch_status(StatusType::PUBLISH_SCN, CHECK_INTERVAL); + } else if (OB_FAIL(get_transfer_task_id(max_transfer_task_id_))) { + LOG_WARN("failed to get_transfer_task_id", KR(ret), KPC(this)); + } else { + switch_status(StatusType::CHECK_END); + } + return ret; +} + +int ObReplicaSafeCheckTask::get_transfer_task_id( + share::ObTransferTaskID &max_task_id) +{ + int ret = OB_SUCCESS; + ObSqlString sql_str; + max_task_id = share::ObTransferTaskID::INVALID_ID; + const int64_t obj_pos = 0; + ObObj result_obj; + if (OB_FAIL(sql_str.assign_fmt("SELECT max(task_id) as max_task_id FROM %s", share::OB_ALL_TRANSFER_TASK_TNAME))) { + LOG_WARN("failed to assign sql", KR(ret), K(sql_str)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, res) + { + common::sqlclient::ObMySQLResult *result = nullptr; + if (!sql_str.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("sql_str is invalid", KR(ret)); + } else if (OB_FAIL(GCTX.sql_proxy_->read(res, MTL_ID(), sql_str.ptr()))) { + LOG_WARN("execute sql failed", KR(ret), K(sql_str)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is null", KR(ret)); + } else if (OB_FAIL(result->next())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("fail to get next", KR(ret)); + } else { + ret = OB_SUCCESS; + } + } else if (OB_FAIL(result->get_obj(obj_pos, result_obj))) { + LOG_WARN("failed to get object", K(ret)); + } else if (result_obj.is_null()) { + ret = OB_SUCCESS; + } else if (OB_UNLIKELY(!result_obj.is_integer_type())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected obj type", K(ret), K(result_obj.get_type())); + } else { + max_task_id = result_obj.get_int(); + } + } + } + return ret; +} + +int ObReplicaSafeCheckTask::check_row_empty( + const ObSqlString &sql_str, + bool &is_empty) +{ + int ret = OB_SUCCESS; + is_empty = false; + SMART_VAR(ObMySQLProxy::MySQLResult, res) + { + common::sqlclient::ObMySQLResult *result = nullptr; + if (!sql_str.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("sql_str is invalid", KR(ret)); + } else if (OB_FAIL(GCTX.sql_proxy_->read(res, MTL_ID(), sql_str.ptr()))) { + LOG_WARN("execute sql failed", KR(ret), K(sql_str)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is null", KR(ret)); + } else if (OB_FAIL(result->next())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("fail to get next", KR(ret)); + } else { + is_empty = true; + ret = OB_SUCCESS; + } + } + } + return ret; +} + +int ObReplicaSafeCheckTask::check_end() +{ + int ret = OB_SUCCESS; + bool no_new_create = false; + bool no_transfer_in = false; + ObSqlString check_new_create_sql; + ObSqlString check_transfer_in_sql; + if (OB_FAIL(check_new_create_sql.assign_fmt("SELECT 1 FROM %s WHERE last_refresh_scn = 0 AND refresh_mode = %ld limit 1", share::OB_ALL_MVIEW_TNAME, ObMVRefreshMode::MAJOR_COMPACTION))) { + LOG_WARN("failed to assign sql", KR(ret), K(check_new_create_sql), KPC(this)); + } else if (OB_FAIL(check_row_empty(check_new_create_sql, no_new_create))) { + LOG_WARN("failed to check_row_empty", KR(ret), K(check_new_create_sql), KPC(this)); + } else if (!no_new_create) { + } else if (!max_transfer_task_id_.is_valid()) { + no_transfer_in = true; + } else if (OB_FAIL(check_transfer_in_sql.assign_fmt("SELECT 1 FROM %s WHERE task_id <= %ld limit 1", share::OB_ALL_TRANSFER_TASK_TNAME, max_transfer_task_id_.id()))) { + LOG_WARN("failed to assign sql", KR(ret), K(check_transfer_in_sql), KPC(this)); + } else if (OB_FAIL(check_row_empty(check_transfer_in_sql, no_transfer_in))) { + LOG_WARN("failed to check_row_empty", KR(ret), K(check_transfer_in_sql), KPC(this)); + } else if (!no_transfer_in) { + } + LOG_INFO("check_end finish", KR(ret), K(no_new_create), K(no_transfer_in), KPC(this)); + if (OB_FAIL(ret)) { + switch_status(StatusType::PUBLISH_SCN, ERROR_RETRY_INTERVAL); + } else if (!no_new_create || !no_transfer_in) { + switch_status(StatusType::CHECK_END, WAIT_END_INTERVAL); + } else { + switch_status(StatusType::NOTICE_SAFE); + } + return ret; +} + +int ObReplicaSafeCheckTask::notice_safe() +{ + int ret = OB_SUCCESS; + // merge version and tenant scn + if (ls_cache_.get_ls_info_cnt() == 0 + || !merge_scn_.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("task ctx is unexpected", KR(ret), KPC(this)); + } else { + ObArray& ls_infos = ls_cache_.get_ls_infos(); + for (int64_t i = 0; OB_SUCC(ret) && i < ls_infos.count(); ++i) { + ObMVMergeSCNInfo &ls_info = ls_infos.at(i); + if (!ls_info.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls_info is invalid", KR(ret), K(ls_info), KPC(this)); + } else if (merge_scn_ != ls_info.major_mv_merge_scn_publish_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("merge_scn is not equal ls_info.major_mv_merge_scn_publish_", KR(ret), K(ls_info), KPC(this)); + } else if (ls_info.major_mv_merge_scn_safe_calc_ < ls_info.major_mv_merge_scn_publish_) { + ObUpdateMergeScnArg arg; + arg.merge_scn_ = merge_scn_; + arg.ls_id_ = ls_info.ls_id_; + if (OB_FAIL(do_multi_trans(transaction::ObTxDataSourceType::MV_NOTICE_SAFE, arg))) { + LOG_WARN("failed to do_multi_trans", KR(ret), K(ls_info), K(arg), K(this)); + } else { + ls_info.major_mv_merge_scn_safe_calc_ = ls_info.major_mv_merge_scn_publish_; + } + } + } + } + LOG_INFO("notice_safe finish", KR(ret), KPC(this)); + + if (OB_LS_LOCATION_LEADER_NOT_EXIST == ret || OB_NOT_MASTER == ret) { + switch_status(StatusType::NOTICE_SAFE, LOCATION_RETRY_INTERVAL); + } else if (OB_FAIL(ret)) { + switch_status(StatusType::PUBLISH_SCN, ERROR_RETRY_INTERVAL); + } else { + switch_status(StatusType::PUBLISH_SCN, CHECK_INTERVAL); + } + return ret; +} + +// void ObReplicaSafeCheckTask::finish() +// { +// int ret = OB_SUCCESS; +// LOG_INFO("replica safe check task finish", KPC(this)); +// // cleanup +// cleanup(); +// // schedule next round +// switch_status(StatusType::PUBLISH_SCN, CHECK_INTERVAL); +// } + +void ObReplicaSafeCheckTask::cleanup() +{ + status_ = StatusType::PUBLISH_SCN; + merge_scn_.reset(); + ls_cache_.reset(); + max_transfer_task_id_.reset(); +} + +} // namespace rootserver +} // namespace oceanbase diff --git a/src/rootserver/mview/ob_replica_safe_check_task.h b/src/rootserver/mview/ob_replica_safe_check_task.h new file mode 100644 index 0000000000..6b384ba490 --- /dev/null +++ b/src/rootserver/mview/ob_replica_safe_check_task.h @@ -0,0 +1,133 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#pragma once + +#include "rootserver/mview/ob_mview_timer_task.h" +#include "share/ob_ls_id.h" +#include "observer/ob_inner_sql_connection.h" +#include "storage/mview/ob_major_mv_merge_info.h" + +namespace oceanbase +{ +namespace rootserver +{ + +struct ObMVMergeSCNInfo +{ + ObMVMergeSCNInfo( + const share::ObLSID &ls_id) + : ls_id_(ls_id), + major_mv_merge_scn_publish_(share::SCN::min_scn()), + major_mv_merge_scn_safe_calc_(share::SCN::min_scn()) + {} + ObMVMergeSCNInfo() + : ls_id_(), + major_mv_merge_scn_publish_(share::SCN::min_scn()), + major_mv_merge_scn_safe_calc_(share::SCN::min_scn()) + {} + bool is_valid() const + { return ls_id_.is_valid() && major_mv_merge_scn_publish_.is_valid() + && major_mv_merge_scn_safe_calc_.is_valid() + && major_mv_merge_scn_publish_ >= major_mv_merge_scn_safe_calc_; } + + + TO_STRING_KV(K_(ls_id), K_(major_mv_merge_scn_publish), K_(major_mv_merge_scn_safe_calc)); + + share::ObLSID ls_id_; + share::SCN major_mv_merge_scn_publish_; + share::SCN major_mv_merge_scn_safe_calc_; +}; + +class ObMVMergeSCNInfoCache +{ +public: + int get_ls_info( + const share::ObLSID &ls_id, + ObMVMergeSCNInfo *&ls_info); + int clear_deleted_ls_info( + const share::SCN &merge_scn); + ObArray& get_ls_infos() + { return arr_; } + + int64_t get_ls_info_cnt() + { return arr_.count(); } + + void reset() + { arr_.reset(); } + + TO_STRING_KV(K(arr_.count()), K_(arr)); + +private: + ObArray arr_; +}; + +class ObReplicaSafeCheckTask : public ObMViewTimerTask +{ +public: + ObReplicaSafeCheckTask(); + virtual ~ObReplicaSafeCheckTask(); + DISABLE_COPY_ASSIGN(ObReplicaSafeCheckTask); + + // for Service + int init(); + int start(); + void stop(); + void wait(); + void destroy(); + + // for TimerTask + void runTimerTask() override; + + TO_STRING_KV(K_(status), K_(in_sched), K_(is_stop), K_(is_inited), K_(merge_scn), K_(max_transfer_task_id), K_(ls_cache)); +private: + static const int64_t CHECK_INTERVAL = 30LL * 1000 * 1000; + static const int64_t ERROR_RETRY_INTERVAL = CHECK_INTERVAL; + static const int64_t WAIT_END_INTERVAL = 5LL * 1000 * 1000; + static const int64_t LOCATION_RETRY_INTERVAL = WAIT_END_INTERVAL; + + enum class StatusType + { + PUBLISH_SCN = 0, + CHECK_END = 1, + NOTICE_SAFE = 2 + }; + +private: + void switch_status(StatusType new_status, int64_t delay = 0); + + int publish_scn(); + int check_end(); + int notice_safe(); + // void finish(); + void cleanup(); + int do_multi_trans( + const transaction::ObTxDataSourceType type, + const ObUpdateMergeScnArg &arg); + int check_row_empty( + const ObSqlString &sql_str, + bool &is_empty); + static int get_transfer_task_id( + share::ObTransferTaskID &max_transfer_task_id); + +private: + StatusType status_; + bool in_sched_; + bool is_stop_; + bool is_inited_; + share::SCN merge_scn_; + share::ObTransferTaskID max_transfer_task_id_; + ObMVMergeSCNInfoCache ls_cache_; +}; + +} // namespace rootserver +} // namespace oceanbase diff --git a/src/rootserver/ob_balance_group_ls_stat_operator.cpp b/src/rootserver/ob_balance_group_ls_stat_operator.cpp index c4e44bfea4..ba4bc277e4 100755 --- a/src/rootserver/ob_balance_group_ls_stat_operator.cpp +++ b/src/rootserver/ob_balance_group_ls_stat_operator.cpp @@ -543,7 +543,7 @@ int ObNewTableTabletAllocator::prepare( if (OB_FAIL(alloc_ls_for_meta_or_sys_tenant_tablet(table_schema))) { LOG_WARN("fail to alloc ls for meta or sys tenant tablet", KR(ret)); } - } else if (table_schema.is_duplicate_table()) { + } else if (table_schema.is_broadcast_table() || table_schema.is_duplicate_table()) { if (OB_FAIL(alloc_ls_for_duplicate_table_(table_schema))) { LOG_WARN("fail to alloc ls for duplicate tablet", KR(ret), K(table_schema)); } diff --git a/src/rootserver/ob_ddl_help.cpp b/src/rootserver/ob_ddl_help.cpp index b9ac0f7521..2b6e916843 100644 --- a/src/rootserver/ob_ddl_help.cpp +++ b/src/rootserver/ob_ddl_help.cpp @@ -94,10 +94,12 @@ int ObTableGroupHelp::add_tables_to_tablegroup(ObMySQLTransaction &trans, ret = OB_OP_NOT_ALLOW; LOG_WARN("sys table's tablegroup should be oceanbase", KR(ret), K(arg), KPC(table_schema)); LOG_USER_ERROR(OB_OP_NOT_ALLOW, "set the tablegroup of system table besides oceanbase"); - } else if (table_schema->has_mlog_table()) { + } else if (table_schema->required_by_mview_refresh()) { ret = OB_NOT_SUPPORTED; - LOG_WARN("alter tablegroup of table with materialized view log is not supported", KR(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter tablegroup of table with materialized view log is"); + LOG_WARN("alter tablegroup of table required by materialized view refresh is not supported", + KR(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, + "alter tablegroup of table required by materialized view refresh is"); } else if (table_schema->is_mlog_table()) { ret = OB_NOT_SUPPORTED; LOG_WARN("alter tablegroup of materialized view log is not supported", KR(ret)); diff --git a/src/rootserver/ob_ddl_operator.cpp b/src/rootserver/ob_ddl_operator.cpp index babbd742cc..db3c2306f6 100644 --- a/src/rootserver/ob_ddl_operator.cpp +++ b/src/rootserver/ob_ddl_operator.cpp @@ -5254,9 +5254,9 @@ int ObDDLOperator::drop_table_to_recyclebin(const ObTableSchema &table_schema, // materialized view will not be dropped into recyclebin if (table_schema.get_table_type() == MATERIALIZED_VIEW) { LOG_WARN("bypass recyclebin for materialized view"); - } else if (OB_UNLIKELY(table_schema.has_mlog_table())) { + } else if (OB_UNLIKELY(table_schema.required_by_mview_refresh())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("table with materialized view log should not come to recyclebin", KR(ret)); + LOG_WARN("table required by materialized view refresh should not come to recyclebin", KR(ret)); } else if (OB_UNLIKELY(table_schema.get_table_type() == MATERIALIZED_VIEW_LOG)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("materialized view log should not come to recyclebin", KR(ret)); diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index f01d4c7aad..09160961e7 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -119,6 +119,7 @@ #include "storage/tx_storage/ob_ls_map.h" #include "storage/tx_storage/ob_ls_service.h" #include "storage/tablelock/ob_lock_inner_connection_util.h" +#include "storage/compaction/ob_compaction_schedule_util.h" #include "share/schema/ob_mview_info.h" #include "storage/mview/ob_mview_sched_job_utils.h" #include "storage/vector_index/ob_vector_index_sched_job_utils.h" @@ -2289,7 +2290,8 @@ int ObDDLService::create_tablets_in_trans_for_mv_(ObIArray &table } else if (OB_ISNULL(GCTX.root_service_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("root service is null", KR(ret)); - } else if (OB_FAIL(ObMajorFreezeHelper::get_frozen_scn(tenant_id, frozen_scn))) { + } else if (OB_FAIL(frozen_scn.convert_for_tx( + compaction::ObBasicMergeScheduler::INIT_COMPACTION_SCN))) { LOG_WARN("failed to get frozen status for create tablet", KR(ret), K(tenant_id)); } else { ObTableCreator table_creator( @@ -3312,7 +3314,9 @@ int ObDDLService::set_raw_table_options( break; } case ObAlterTableArg::DUPLICATE_SCOPE: { - new_table_schema.set_duplicate_scope(alter_table_schema.get_duplicate_scope()); + // alter table duplicate scope not allowed in master now + new_table_schema.set_duplicate_attribute(alter_table_schema.get_duplicate_scope(), + alter_table_schema.get_duplicate_read_consistency()); break; } case ObAlterTableArg::ENABLE_ROW_MOVEMENT: { @@ -6991,6 +6995,11 @@ int ObDDLService::alter_table_index(obrpc::ObAlterTableArg &alter_table_arg, continue; } } + if (OB_SUCC(ret) && new_table_schema.mv_major_refresh()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support to add index on mv", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "add index on major refresh materialized view is"); + } if (OB_FAIL(ret)) { } else if (create_index_arg->index_name_.empty()) { if (OB_FAIL(generate_index_name(*create_index_arg, @@ -14586,10 +14595,14 @@ int ObDDLService::check_is_offline_ddl(ObAlterTableArg &alter_table_arg, bool is_adding_constraint = false; bool is_column_store = false; uint64_t table_id = alter_table_arg.alter_table_schema_.get_table_id(); - if (orig_table_schema->has_mlog_table()) { + if (orig_table_schema->required_by_mview_refresh()) { ret = OB_NOT_SUPPORTED; - LOG_WARN("double table long running ddl on table with materialized view log is not supported", KR(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "double table long running ddl on table with materialized view log is"); + LOG_WARN("double table long running ddl on table required by materialized view refresh is " + "not supported", + KR(ret)); + LOG_USER_ERROR( + OB_NOT_SUPPORTED, + "double table long running ddl on table required by materialized view refresh is"); } else if (orig_table_schema->is_mlog_table()) { ret = OB_NOT_SUPPORTED; LOG_WARN("double table long running ddl on materialized view log is not supported", KR(ret)); @@ -16034,6 +16047,15 @@ int ObDDLService::get_and_check_table_schema( if (OB_SUCC(ret) && !is_alter_pk) { allow_alter_mview = true; } + } else { + // only allow for alter tablegroup + if (!alter_table_arg.is_alter_indexs_ && !alter_table_arg.is_alter_columns_ && !alter_table_arg.is_alter_partitions_ && + !alter_table_arg.is_update_global_indexes_ && !alter_table_arg.is_convert_to_character_ && alter_table_arg.is_alter_options_) { + if (1 == alter_table_arg.alter_table_schema_.alter_option_bitset_.num_members() && + alter_table_arg.alter_table_schema_.alter_option_bitset_.has_member(ObAlterTableArg::TABLEGROUP_NAME)) { + allow_alter_mview = true; + } + } } if (OB_SUCC(ret) && !allow_alter_mview) { ret = OB_NOT_SUPPORTED; @@ -17738,14 +17760,13 @@ int ObDDLService::rename_table(const obrpc::ObRenameTableArg &rename_table_arg) LOG_WARN("rename materialized view log is not supported", KR(ret), K(table_schema->get_table_name())); LOG_USER_ERROR(OB_NOT_SUPPORTED, "rename materialized view log is"); - } else if (table_schema->has_mlog_table()) { + } else if (table_schema->required_by_mview_refresh()) { ret = OB_NOT_SUPPORTED; - LOG_WARN("rename table with materialized view log is not supported", + LOG_WARN("rename table required by materialized view refresh is not supported", KR(ret), K(table_schema->get_table_name())); LOG_USER_ERROR(OB_NOT_SUPPORTED, "rename table with materialized view log is"); - } else if (OB_FAIL(ObDependencyInfo::collect_all_dep_objs(tenant_id, - table_schema->get_table_id(), - trans, all_dep_objs))) { + } else if (OB_FAIL(ObDependencyInfo::collect_all_dep_objs( + tenant_id, table_schema->get_table_id(), trans, all_dep_objs))) { LOG_WARN("failed to collect dep info", K(ret)); } } @@ -21425,7 +21446,11 @@ int ObDDLService::swap_orig_and_hidden_table_state(obrpc::ObAlterTableArg &alter } if (OB_SUCC(ret)) { int64_t schema_version = table_schemas[table_schemas.count()-1].get_schema_version(); - if (OB_FAIL(unbind_hidden_tablets(*orig_table_schema, *hidden_table_schema, + if (orig_table_schema->mv_major_refresh()) { + // for major refresh mv skip modify ddl_info + // because of we read majoar mv data by specified snapshot + // last_refresh_scn may less then ddl snapshot + } else if (OB_FAIL(unbind_hidden_tablets(*orig_table_schema, *hidden_table_schema, schema_version, trans))) { LOG_WARN("failed to unbind hidden tablets", K(ret)); } @@ -23418,12 +23443,13 @@ int ObDDLService::check_table_schema_is_legal(const ObDatabaseSchema & database_ LOG_WARN("can not truncate table in recyclebin", KR(ret), K(table_name), K(table_id), K(database_name)); } else if (table_schema.is_user_table() || table_schema.is_mysql_tmp_table()) { - if (table_schema.has_mlog_table()) { + if (table_schema.required_by_mview_refresh()) { ret = OB_NOT_SUPPORTED; - LOG_WARN("truncate table with materialized view log is not supported", + LOG_WARN("truncate table required by materialized view refresh is not supported", KR(ret), K(table_schema), K(table_id)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "truncate table with materialized view log is"); - } else if (check_foreign_key && OB_FAIL(check_is_foreign_key_parent_table(table_schema, trans))){ + LOG_USER_ERROR(OB_NOT_SUPPORTED, "truncate table required by materialized view refresh is"); + } else if (check_foreign_key && + OB_FAIL(check_is_foreign_key_parent_table(table_schema, trans))) { LOG_WARN("failed to check table is foreign key's parent table", KR(ret), K(table_name), K(table_id)); } } else if (0 != table_schema.get_autoinc_column_id()) { @@ -23669,9 +23695,9 @@ int ObDDLService::truncate_table(const ObTruncateTableArg &arg, } } else if (OB_FAIL(check_enable_sys_table_ddl(*orig_table_schema, OB_DDL_TRUNCATE_TABLE_CREATE))) { LOG_WARN("ddl is not allowed on system table", K(ret)); - } else if (orig_table_schema->has_mlog_table()) { + } else if (orig_table_schema->required_by_mview_refresh()) { ret = OB_NOT_SUPPORTED; - LOG_WARN("truncate table with materialized view log is not supported", + LOG_WARN("truncate table required by materialized view refresh is not supported", KR(ret), KPC(orig_table_schema)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "truncate table with materialized view log is"); } else if (!orig_table_schema->check_can_do_ddl()) { @@ -24307,15 +24333,15 @@ int ObDDLService::create_table_like(const ObCreateTableLikeArg &arg) LOG_USER_ERROR(OB_ERR_WRONG_OBJECT, to_cstring(arg.origin_db_name_), to_cstring(arg.origin_table_name_), "BASE TABLE"); LOG_WARN("create table like inner table not allowed", K(ret), K(arg)); - } else if (orig_table_schema->has_mlog_table()) { + } else if (orig_table_schema->required_by_mview_refresh()) { ret = OB_NOT_SUPPORTED; - LOG_WARN("create table like on table with materialized view log is not supported", KR(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "create table like on table with materialized view log is"); - } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, - arg.new_db_name_, - arg.new_table_name_, - false, - new_table_schema))) { + LOG_WARN( + "create table like on table required by materialized view refresh is not supported", + KR(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, + "create table like on table required by materialized view refresh is"); + } else if (OB_FAIL(schema_guard.get_table_schema( + tenant_id, arg.new_db_name_, arg.new_table_name_, false, new_table_schema))) { } else if (NULL != new_table_schema) { ret = OB_ERR_TABLE_EXIST; LOG_WARN("target table already exist", K(arg), K(tenant_id), K(ret)); @@ -26318,10 +26344,10 @@ int ObDDLService::drop_table(const ObDropTableArg &drop_table_arg, const obrpc:: } else if (!drop_table_arg.force_drop_ && table_schema->is_in_recyclebin()) { ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT; LOG_WARN("can not drop table in recyclebin, use purge instead", K(ret), K(table_item)); - } else if (table_schema->has_mlog_table()) { + } else if (table_schema->required_by_mview_refresh() && !table_schema->is_index_table()) { ret = OB_NOT_SUPPORTED; - LOG_WARN("drop table with materialized view log is not supported", KR(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "drop table with materialized view log is"); + LOG_WARN("drop table required by materialized view refresh is not supported", KR(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "drop table required by materialized view refresh is"); } else if (OB_FAIL(tmp_table_schema.assign(*table_schema))) { LOG_WARN("fail to assign table schema", K(ret)); } else if (OB_FAIL(schema_guard.check_database_in_recyclebin( diff --git a/src/rootserver/ob_index_builder.cpp b/src/rootserver/ob_index_builder.cpp index bb9a73737a..cad2ed0683 100644 --- a/src/rootserver/ob_index_builder.cpp +++ b/src/rootserver/ob_index_builder.cpp @@ -1483,7 +1483,8 @@ int ObIndexBuilder::generate_schema( const bool need_generate_index_schema_column = (is_index_local_storage || global_index_without_column_info); schema.set_table_mode(data_schema.get_table_mode_flag()); schema.set_table_state_flag(data_schema.get_table_state_flag()); - schema.set_duplicate_scope(data_schema.get_duplicate_scope()); + schema.set_mv_mode(data_schema.get_mv_mode()); + schema.set_duplicate_attribute(data_schema.get_duplicate_scope(), data_schema.get_duplicate_read_consistency()); if (OB_FAIL(set_basic_infos(arg, data_schema, schema))) { LOG_WARN("set_basic_infos failed", K(arg), K(data_schema), K(ret)); } else if (need_generate_index_schema_column diff --git a/src/rootserver/ob_lob_meta_builder.cpp b/src/rootserver/ob_lob_meta_builder.cpp index 4acdf96a4e..77fd5b74cf 100644 --- a/src/rootserver/ob_lob_meta_builder.cpp +++ b/src/rootserver/ob_lob_meta_builder.cpp @@ -160,7 +160,8 @@ int ObLobMetaBuilder::set_basic_infos( aux_lob_meta_schema.set_pctfree(data_schema.get_pctfree()); aux_lob_meta_schema.set_storage_format_version(data_schema.get_storage_format_version()); aux_lob_meta_schema.set_progressive_merge_round(data_schema.get_progressive_merge_round()); - aux_lob_meta_schema.set_duplicate_scope(data_schema.get_duplicate_scope()); + aux_lob_meta_schema.set_duplicate_attribute(data_schema.get_duplicate_scope(), + data_schema.get_duplicate_read_consistency()); if (OB_FAIL(aux_lob_meta_schema.set_compress_func_name(data_schema.get_compress_func_name()))) { LOG_WARN("set_compress_func_name failed", K(data_schema)); } diff --git a/src/rootserver/ob_lob_piece_builder.cpp b/src/rootserver/ob_lob_piece_builder.cpp index 0cbcb349c0..0639ab8a7b 100644 --- a/src/rootserver/ob_lob_piece_builder.cpp +++ b/src/rootserver/ob_lob_piece_builder.cpp @@ -159,7 +159,7 @@ int ObLobPieceBuilder::set_basic_infos( aux_lob_piece_schema.set_pctfree(data_schema.get_pctfree()); aux_lob_piece_schema.set_storage_format_version(data_schema.get_storage_format_version()); aux_lob_piece_schema.set_progressive_merge_round(data_schema.get_progressive_merge_round()); - aux_lob_piece_schema.set_duplicate_scope(data_schema.get_duplicate_scope()); + aux_lob_piece_schema.set_duplicate_attribute(data_schema.get_duplicate_scope(), data_schema.get_duplicate_read_consistency()); if (OB_FAIL(aux_lob_piece_schema.set_compress_func_name(data_schema.get_compress_func_name()))) { LOG_WARN("set_compress_func_name failed", K(data_schema)); } diff --git a/src/rootserver/ob_mlog_builder.cpp b/src/rootserver/ob_mlog_builder.cpp index 12b53f4a1a..9263a3da65 100644 --- a/src/rootserver/ob_mlog_builder.cpp +++ b/src/rootserver/ob_mlog_builder.cpp @@ -645,7 +645,8 @@ int ObMLogBuilder::set_basic_infos( mlog_schema.set_table_mode_flag(ObTableModeFlag::TABLE_MODE_QUEUING_EXTREME); mlog_schema.set_table_type(MATERIALIZED_VIEW_LOG); mlog_schema.set_index_status(ObIndexStatus::INDEX_STATUS_UNAVAILABLE); - mlog_schema.set_duplicate_scope(base_table_schema.get_duplicate_scope()); + mlog_schema.set_duplicate_attribute(base_table_schema.get_duplicate_scope(), + base_table_schema.get_duplicate_read_consistency()); mlog_schema.set_table_state_flag(base_table_schema.get_table_state_flag()); mlog_schema.set_table_id(create_mlog_arg.mlog_table_id_); mlog_schema.set_data_table_id(base_table_schema.get_table_id()); diff --git a/src/rootserver/ob_root_service.cpp b/src/rootserver/ob_root_service.cpp index c102534b93..07cfe74097 100644 --- a/src/rootserver/ob_root_service.cpp +++ b/src/rootserver/ob_root_service.cpp @@ -10634,7 +10634,7 @@ int ObRootService::table_allow_ddl_operation(const obrpc::ObAlterTableArg &arg) LOG_WARN("try to alter invisible table schema", K(schema->get_session_id()), K(arg)); LOG_USER_ERROR(OB_OP_NOT_ALLOW, "try to alter invisible table"); } - } else if (schema->has_mlog_table() || schema->is_mlog_table()) { + } else if (schema->required_by_mview_refresh() || schema->is_mlog_table()) { if (OB_FAIL(ObResolverUtils::check_allowed_alter_operations_for_mlog( tenant_id, arg, *schema))) { LOG_WARN("failed to check allowed alter operation for mlog", diff --git a/src/rootserver/ob_tablet_creator.cpp b/src/rootserver/ob_tablet_creator.cpp index a30f883de8..726a657cf0 100644 --- a/src/rootserver/ob_tablet_creator.cpp +++ b/src/rootserver/ob_tablet_creator.cpp @@ -229,7 +229,7 @@ int ObBatchCreateTabletHelper::add_table_schema_( LOG_WARN("failed to allocate storage schema", KR(ret), K(table_schema)); } else if (FALSE_IT(create_tablet_schema = new (create_tablet_schema_ptr)ObCreateTabletSchema())) { } else if (OB_FAIL(create_tablet_schema->init(batch_arg_.allocator_, table_schema, compat_mode, - false/*skip_column_info*/, tenant_data_version < DATA_VERSION_4_3_0_0 ? ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V2 : ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3))) { + false/*skip_column_info*/, tenant_data_version < DATA_VERSION_4_3_0_0 ? ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V2 : ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_LATEST))) { LOG_WARN("failed to init storage schema", KR(ret), K(table_schema)); } else if (OB_FAIL(batch_arg_.create_tablet_schemas_.push_back(create_tablet_schema))) { LOG_WARN("failed to push back table schema", KR(ret), K(table_schema)); diff --git a/src/rootserver/restore/ob_restore_scheduler.cpp b/src/rootserver/restore/ob_restore_scheduler.cpp index 5702380f00..98db728413 100644 --- a/src/rootserver/restore/ob_restore_scheduler.cpp +++ b/src/rootserver/restore/ob_restore_scheduler.cpp @@ -42,6 +42,10 @@ #include "share/ob_master_key_getter.h" #endif #include "share/backup/ob_backup_connectivity.h" +#include "share/ob_global_stat_proxy.h" // ObGlobalStatProxy +#include "rootserver/mview/ob_mview_timer_task.h" +#include "rootserver/mview/ob_collect_mv_merge_info_task.h" +#include "rootserver/mview/ob_mview_push_refresh_scn_task.h" namespace oceanbase { @@ -970,12 +974,13 @@ int ObRestoreScheduler::restore_upgrade(const ObPhysicalRestoreJob &job_info) LOG_WARN("invalid tenant id", KR(ret), K(tenant_id_)); } else if (OB_FAIL(restore_service_->check_stop())) { LOG_WARN("restore scheduler stopped", KR(ret)); + } else if (OB_FAIL(wait_restore_safe_mview_merge_info_())) { + LOG_WARN("fail to wait restore safe mview merge info, need retry", + K(ret), K(job_info)); } else { - if (OB_SUCC(ret)) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = try_update_job_status(*sql_proxy_, ret, job_info))) { - LOG_WARN("fail to update job status", K(ret), K(tmp_ret), K(job_info)); - } + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = try_update_job_status(*sql_proxy_, ret, job_info))) { + LOG_WARN("fail to update job status", K(ret), K(tmp_ret), K(job_info)); } } LOG_INFO("[RESTORE] upgrade pre finish", KR(ret), K(job_info)); @@ -1829,5 +1834,102 @@ int ObRestoreScheduler::update_tenant_restore_data_mode_(const uint64_t tenant_i return ret; } +int ObRestoreScheduler::try_collect_ls_mv_merge_scn_(const share::SCN &major_mv_merge_scn) +{ + int ret = OB_SUCCESS; + ObLSAttrArray ls_attr_array; + const uint64_t user_tenant_id = gen_user_tenant_id(MTL_ID()); + share::ObLSAttrOperator ls_attr_operator(user_tenant_id, sql_proxy_); + if (!is_valid_tenant_id(user_tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant_id!", K(ret), K(user_tenant_id)); + } else if (OB_FAIL(ls_attr_operator.get_all_ls_by_order(ls_attr_array))) { + LOG_WARN("fail to get all ls", KR(ret)); + } else { + share::SCN min_merge_scn(share::SCN::max_scn()); + share::SCN merge_scn; + ARRAY_FOREACH(ls_attr_array, i) { + const ObLSAttr &ls_attr = ls_attr_array.at(i); + if (ls_attr.get_ls_id().is_sys_ls()) { + // skip sys ls + } else if (OB_FAIL(ObCollectMvMergeInfoTask:: + collect_ls_member_merge_info(user_tenant_id, ls_attr.get_ls_id(), merge_scn))) { + LOG_WARN("fail to collect ls member merge scn", KR(ret), K(ls_attr), K(user_tenant_id)); + } else if (min_merge_scn > merge_scn) { + min_merge_scn = merge_scn; + } + } + if (OB_SUCC(ret)) { + if (min_merge_scn >= major_mv_merge_scn) { + // do nothing + } else { + ret = OB_EAGAIN; + LOG_WARN("ls member merge scn is less than lastest merge_scn", + K(ret), K(min_merge_scn), K(tenant_id_), K(major_mv_merge_scn)); + } + } + } + return ret; +} + +ERRSIM_POINT_DEF(ERRSIM_WAIT_RESTORE_SAFE_MVIEW); +// step 1 get major tenant mv merge scn +// step 2 gWAIT_RESTORE_SFAFE_MVIEWs +// step 3 check major mv merge scn is geater than tenant mv merge scn +// step 4 safe check +int ObRestoreScheduler::wait_restore_safe_mview_merge_info_() +{ + int ret = OB_SUCCESS; + + bool need_schedule = false; + share::SCN mv_lastest_merge_scn(share::SCN::min_scn()); +#ifdef ERRSIM + ret = ERRSIM_WAIT_RESTORE_SAFE_MVIEW ? : OB_SUCCESS; + if (OB_FAIL(ret)) { + LOG_INFO("error sim to wait in restore upgrade status", K(ret), K(tenant_id_)); + } +#endif + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error! sql proxy is null!", K(ret), K(sql_proxy_)); + } else if (OB_FAIL(ObMViewTimerTask:: + need_schedule_major_refresh_mv_task(tenant_id_, need_schedule))) { + LOG_WARN("failed to check need schedule", KR(ret), K(tenant_id_)); + } else if (!need_schedule) { + // do nothing + } else { + share::SCN major_mv_merge_scn(share::SCN::min_scn()); + ObGlobalStatProxy global_proxy(*sql_proxy_, tenant_id_); + if (OB_FAIL(OB_FAIL(ObCollectMvMergeInfoTask:: + get_min_mv_tablet_major_compaction_scn(mv_lastest_merge_scn)))) { + LOG_WARN("fail to mv tablet merge scn", K(ret), K(tenant_id_)); + } else if (OB_FAIL(global_proxy.get_major_refresh_mv_merge_scn(false, /*for update*/ + major_mv_merge_scn))) { + LOG_WARN("fail to get major_refresh_mv_merge_scn", K(ret), K(tenant_id_)); + } else if (OB_FAIL(ObMViewTimerTask::check_mview_last_refresh_scn(tenant_id_, + major_mv_merge_scn))) { + LOG_WARN("fail to check mview last refesh scn", K(ret), K(tenant_id_), K(major_mv_merge_scn)); + } + if (OB_SUCC(ret)) { + if (mv_lastest_merge_scn < major_mv_merge_scn) { + ret = OB_EAGAIN; + LOG_INFO("major_refresh_mv_merge_scn is not greater than tenant mv merge scn", + K(ret), K(tenant_id_), K(mv_lastest_merge_scn), K(major_mv_merge_scn)); + } else { + LOG_INFO("major_merge_scn is greater than tenant_mv_merge_scn, pass check", + K(ret), K(tenant_id_), K(mv_lastest_merge_scn), K(major_mv_merge_scn)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(try_collect_ls_mv_merge_scn_(major_mv_merge_scn))) { + LOG_WARN("fail to collect ls mv merge scn", K(ret), K(major_mv_merge_scn)); + } else if (FALSE_IT(void(ObMViewPushRefreshScnTask:: + check_major_mv_refresh_scn_safety(tenant_id_)))) { + } + } + } + return ret; +} } // end namespace rootserver } // end namespace oceanbase diff --git a/src/rootserver/restore/ob_restore_scheduler.h b/src/rootserver/restore/ob_restore_scheduler.h index cc588121ae..b6c44f5273 100644 --- a/src/rootserver/restore/ob_restore_scheduler.h +++ b/src/rootserver/restore/ob_restore_scheduler.h @@ -119,6 +119,8 @@ private: int update_tenant_restore_data_mode_to_normal_(const uint64_t tenant_id); int update_tenant_restore_data_mode_(const uint64_t tenant_id, const share::ObRestoreDataMode &new_restore_data_mode); int wait_sys_job_ready_(const ObPhysicalRestoreJob &job, bool &is_ready); + int wait_restore_safe_mview_merge_info_(); + int try_collect_ls_mv_merge_scn_(const share::SCN &tenant_mv_merge_scn); int update_restore_progress_by_bytes_(const ObPhysicalRestoreJob &job, const int64_t total_bytes, const int64_t finish_bytes); private: bool inited_; diff --git a/src/rootserver/standby/ob_standby_service.cpp b/src/rootserver/standby/ob_standby_service.cpp index 84ecfaaba6..594a44c749 100644 --- a/src/rootserver/standby/ob_standby_service.cpp +++ b/src/rootserver/standby/ob_standby_service.cpp @@ -34,6 +34,7 @@ #include "storage/tx/ob_timestamp_service.h" // ObTimestampService #include "storage/high_availability/ob_transfer_lock_utils.h" // ObMemberListLockUtils #include "common/ob_tenant_data_version_mgr.h" +#include "share/schema/ob_mview_info.h" #include "logservice/restoreservice/ob_log_restore_handler.h"//RestoreSyncStatus namespace oceanbase @@ -448,6 +449,7 @@ int ObStandbyService::do_recover_tenant( ObTenantStatus tenant_status = TENANT_STATUS_MAX; ObLSRecoveryStatOperator ls_recovery_operator; ObLSRecoveryStat sys_ls_recovery; + bool is_contains_new_mv_tenant = false; if (OB_FAIL(check_inner_stat_())) { LOG_WARN("inner stat error", KR(ret), K_(inited)); } else if (!obrpc::ObRecoverTenantArg::is_valid(recover_type, recovery_until_scn) @@ -462,6 +464,15 @@ int ObStandbyService::do_recover_tenant( LOG_WARN("invalid argument", KR(ret), K(tenant_id)); } else if (OB_FAIL(get_tenant_status(tenant_id, tenant_status))) { LOG_WARN("failed to get tenant status", KR(ret), K(tenant_id)); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_version))) { + LOG_WARN("failed to get tenant min version", KR(ret), K(tenant_id)); + } else if (tenant_version >= DATA_VERSION_4_3_4_0 + && OB_FAIL(ObMViewInfo::contains_major_refresh_mview_in_creation(*sql_proxy_, tenant_id, is_contains_new_mv_tenant))) { + LOG_WARN("failed to contains_major_refresh_mview_in_creation", KR(ret), K(tenant_id), K(tenant_version)); + } else if (is_contains_new_mv_tenant) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("there is new mv in this tenant", K(tenant_id)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "there is new mv in this tenant, recover is"); } else if (OB_FAIL(trans.start(sql_proxy_, exec_tenant_id))) { LOG_WARN("failed to start trans", KR(ret), K(exec_tenant_id), K(tenant_id)); } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id, &trans, true, tenant_info))) { diff --git a/src/share/backup/ob_backup_data_table_operator.cpp b/src/share/backup/ob_backup_data_table_operator.cpp index 4cc09ecf08..34e20a308e 100644 --- a/src/share/backup/ob_backup_data_table_operator.cpp +++ b/src/share/backup/ob_backup_data_table_operator.cpp @@ -19,6 +19,7 @@ #include "share/config/ob_server_config.h" #include "lib/mysqlclient/ob_mysql_transaction.h" #include "share/ob_share_util.h" +#include "lib/ob_define.h" namespace oceanbase { @@ -2432,6 +2433,62 @@ int ObBackupSkippedTabletOperator::move_skip_tablet_to_his( return ret; } +int ObBackupMViewOperator::get_all_major_compaction_mview_dep_tablet_list(common::ObMySQLProxy &proxy, + const uint64_t tenant_id, + const share::SCN &snapshot, + common::ObIArray &tablet_list) +{ + int ret = OB_SUCCESS; + tablet_list.reset(); + ObSqlString sql; + int64_t affected_rows = -1; + if (OB_INVALID_TENANT_ID == tenant_id || !snapshot.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid arg", K(ret), K(tenant_id), K(snapshot)); + } else { + HEAP_VAR(ObMySQLProxy::ReadResult, res) { + ObMySQLResult *result = NULL; + if (OB_FAIL(sql.assign_fmt("select tablet_id from %s as of snapshot %ld where table_id in (" + " select p_obj from %s as of snapshot %ld" + " where mview_id in (select mview_id from %s as of snapshot %ld where refresh_mode=%ld) " + " union all " + " select data_table_id from %s as of snapshot %ld" + " where table_id in (select mview_id from %s as of snapshot %ld where refresh_mode=%ld))", + OB_ALL_TABLET_TO_LS_TNAME, snapshot.get_val_for_inner_table_field(), + OB_ALL_MVIEW_DEP_TNAME, snapshot.get_val_for_inner_table_field(), + OB_ALL_MVIEW_TNAME, snapshot.get_val_for_inner_table_field(), schema::ObMVRefreshMode::MAJOR_COMPACTION, + OB_ALL_TABLE_TNAME, snapshot.get_val_for_inner_table_field(), + OB_ALL_MVIEW_TNAME, snapshot.get_val_for_inner_table_field(), schema::ObMVRefreshMode::MAJOR_COMPACTION))) { + LOG_WARN("failed to assign fmt", K(ret), K(snapshot)); + } else if (OB_FAIL(proxy.read(res, common::gen_user_tenant_id(tenant_id), sql.ptr()))) { + LOG_WARN("failed to read sql", K(ret), K(tenant_id), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is null", K(ret), K(sql)); + } else { + while (OB_SUCC(ret)) { + if (OB_FAIL(result->next())) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + break; + } else { + LOG_WARN("failed to get next row", K(ret)); + } + } else { + int64_t tablet_id = -1; + EXTRACT_INT_FIELD_MYSQL(*result, OB_STR_TABLET_ID, tablet_id, int64_t); + if (FAILEDx(tablet_list.push_back(ObTabletID(tablet_id)))) { + LOG_WARN("failed to push back", K(ret), K(tablet_id)); + } + } + } + } + } + } + LOG_INFO("get all mview dep tablet list", K(ret), K(sql), K(tablet_list)); + return ret; +} + /* *---------------------------__all_backup_ls_task_info------------------------ */ diff --git a/src/share/backup/ob_backup_data_table_operator.h b/src/share/backup/ob_backup_data_table_operator.h index b910a31edc..a85e145869 100644 --- a/src/share/backup/ob_backup_data_table_operator.h +++ b/src/share/backup/ob_backup_data_table_operator.h @@ -58,6 +58,14 @@ private: static int do_parse_skip_tablet_result_(sqlclient::ObMySQLResult &result, ObBackupSkipTabletAttr &backup_set_desc); }; +class ObBackupMViewOperator { +public: + static int get_all_major_compaction_mview_dep_tablet_list(common::ObMySQLProxy &proxy, + const uint64_t tenant_id, + const share::SCN &snapshot, + common::ObIArray &tablet_list); +}; + class ObBackupSetFileOperator : public ObBackupBaseTableOperator { public: diff --git a/src/share/backup/ob_backup_path.cpp b/src/share/backup/ob_backup_path.cpp index 9fd672e4a9..0c9a4bbfef 100644 --- a/src/share/backup/ob_backup_path.cpp +++ b/src/share/backup/ob_backup_path.cpp @@ -635,6 +635,26 @@ int ObBackupPath::join_table_list_meta_info_file(const share::SCN &scn) return ret; } +int ObBackupPath::join_major_compaction_mview_dep_tablet_list_file() +{ + int ret = OB_SUCCESS; + char file_name[OB_MAX_BACKUP_PATH_LENGTH] = { 0 }; + if (cur_pos_ <= 0) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret), K(*this)); + } else if (OB_FAIL(databuff_printf(file_name, + sizeof(file_name), + "%s", + OB_STR_MAJOR_COMPACTION_MVIEW_DEP_TABLET_LIST))) { + LOG_WARN("failed to join table list tmp file", K(ret), K(*this)); + } else if (OB_FAIL(join(file_name, ObBackupFileSuffix::BACKUP))) { + LOG_WARN("failed to join file_name", K(ret), K(file_name)); + } else if (OB_FAIL(trim_right_backslash())) { + LOG_WARN("failed to trim right backslash", K(ret)); + } + return ret; +} + // param case: entry_d_name -> 'checkpoint_info.1678226622262333112.obarc', file_name -> 'checkpoint_info', type -> ARCHIVE // result : checkpoint -> 1678226622262333112 int ObBackupPath::parse_checkpoint(const char *entry_d_name, const common::ObString &file_name, const ObBackupFileSuffix &type, uint64_t &checkpoint) @@ -1571,6 +1591,17 @@ int ObBackupPathUtil::get_table_list_part_file_path(const share::ObBackupDest &b return ret; } +// file:///obbackup/backup_set_1_full/infos/major_compaction_mview_dep_tablet_list +int ObBackupPathUtil::get_major_compaction_mview_dep_tablet_list_path(const share::ObBackupDest &backup_set_dest, share::ObBackupPath &path) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(get_ls_info_dir_path(backup_set_dest, path))) { + LOG_WARN("fail to get backup set info path", K(ret), K(backup_set_dest)); + } else if (OB_FAIL(path.join_major_compaction_mview_dep_tablet_list_file())) { + LOG_WARN("failed to join major compaction mview dep tablet list file", K(ret)); + } + return ret; +} int ObBackupPathUtil::construct_backup_set_dest(const share::ObBackupDest &backup_tenant_dest, const share::ObBackupSetDesc &backup_desc, share::ObBackupDest &backup_set_dest) diff --git a/src/share/backup/ob_backup_path.h b/src/share/backup/ob_backup_path.h index 9d2b471516..c71725df28 100644 --- a/src/share/backup/ob_backup_path.h +++ b/src/share/backup/ob_backup_path.h @@ -63,6 +63,7 @@ public: int join_table_list_dir(); int join_table_list_part_file(const share::SCN &scn, const int64_t part_no); int join_table_list_meta_info_file(const share::SCN &scn); + int join_major_compaction_mview_dep_tablet_list_file(); static int parse_checkpoint(const char *entry_d_name, const common::ObString &file_name, const ObBackupFileSuffix &type, uint64_t &checkpoint); static int parse_partial_table_list_file_name(const char *entry_d_name, const share::SCN &scn, int64_t &part_no); static int parse_table_list_meta_file_name(const char *entry_d_name, share::SCN &scn); @@ -287,6 +288,10 @@ struct ObBackupPathUtil // file:///obbackup/backup_set_1_full/infos/table_list/table_list.[scn].[part_no].obbak static int get_table_list_part_file_path(const share::ObBackupDest &backup_set_dest, const share::SCN &scn, const int64_t part_no, share::ObBackupPath &path); + + // file:///obbackup/backup_set_1_full/infos/major_compaction_mview_dep_tablet_list + static int get_major_compaction_mview_dep_tablet_list_path(const share::ObBackupDest &backup_set_dest, share::ObBackupPath &path); + static int construct_backup_set_dest(const share::ObBackupDest &backup_tenant_dest, const share::ObBackupSetDesc &backup_desc, share::ObBackupDest &backup_set_dest); static int construct_backup_complement_log_dest(const share::ObBackupDest &backup_tenant_dest, diff --git a/src/share/backup/ob_backup_struct.cpp b/src/share/backup/ob_backup_struct.cpp index 439a412db7..9247cf8687 100644 --- a/src/share/backup/ob_backup_struct.cpp +++ b/src/share/backup/ob_backup_struct.cpp @@ -2078,6 +2078,7 @@ bool ObBackupUtils::is_need_retry_error(const int err) case OB_TABLET_NOT_EXIST : case OB_CHECKSUM_ERROR : case OB_VERSION_NOT_MATCH: + case OB_INVALID_DATA: bret = false; break; default: diff --git a/src/share/backup/ob_backup_struct.h b/src/share/backup/ob_backup_struct.h index 33b086baf1..9712d220c8 100644 --- a/src/share/backup/ob_backup_struct.h +++ b/src/share/backup/ob_backup_struct.h @@ -431,6 +431,7 @@ const char *const OB_STR_MAX_BANDWIDTH = "max_bandwidth"; const char *const OB_STR_MAX_IOPS_AND_MAX_BANDWIDTH = "max_iops, max_bandwidth"; const char *const OB_STR_TABLE_LIST = "table_list"; const char *const OB_STR_TABLE_LIST_META_INFO = "table_list_meta_info"; +const char *const OB_STR_MAJOR_COMPACTION_MVIEW_DEP_TABLET_LIST = "major_compaction_mview_dep_tablet_list"; enum ObBackupFileType { @@ -474,6 +475,7 @@ enum ObBackupFileType BACKUP_TABLET_METAS_INFO = 37, BACKUP_TABLE_LIST_FILE = 38, BACKUP_TABLE_LIST_META_FILE = 39, + BACKUP_MVIEW_DEP_TABLET_LIST_FILE = 40, // type <=255 is write header struct to disk directly // type > 255 is use serialization to disk BACKUP_MAX_DIRECT_WRITE_TYPE = 255, diff --git a/src/share/inner_table/ob_inner_table_schema.101_150.cpp b/src/share/inner_table/ob_inner_table_schema.101_150.cpp index acc09a933b..dd8cad6d17 100644 --- a/src/share/inner_table/ob_inner_table_schema.101_150.cpp +++ b/src/share/inner_table/ob_inner_table_schema.101_150.cpp @@ -6985,6 +6985,25 @@ int ObInnerTableSchema::all_table_history_schema(ObTableSchema &table_schema) micro_index_clustered_default, micro_index_clustered_default); //default_value } + + if (OB_SUCC(ret)) { + ObObj mv_mode_default; + mv_mode_default.set_int(0); + ADD_COLUMN_SCHEMA_T("mv_mode", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false, //is_autoincrement + mv_mode_default, + mv_mode_default); //default_value + } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema.11001_11050.cpp b/src/share/inner_table/ob_inner_table_schema.11001_11050.cpp index db489bd744..6cf664bbd3 100644 --- a/src/share/inner_table/ob_inner_table_schema.11001_11050.cpp +++ b/src/share/inner_table/ob_inner_table_schema.11001_11050.cpp @@ -14639,6 +14639,25 @@ int ObInnerTableSchema::all_virtual_core_all_table_schema(ObTableSchema &table_s micro_index_clustered_default, micro_index_clustered_default); //default_value } + + if (OB_SUCC(ret)) { + ObObj mv_mode_default; + mv_mode_default.set_int(0); + ADD_COLUMN_SCHEMA_T("mv_mode", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + mv_mode_default, + mv_mode_default); //default_value + } table_schema.set_index_using_type(USING_HASH); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema.12051_12100.cpp b/src/share/inner_table/ob_inner_table_schema.12051_12100.cpp index d0373d10ba..4a4650329b 100644 --- a/src/share/inner_table/ob_inner_table_schema.12051_12100.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12051_12100.cpp @@ -6899,6 +6899,25 @@ int ObInnerTableSchema::all_virtual_table_schema(ObTableSchema &table_schema) micro_index_clustered_default, micro_index_clustered_default); //default_value } + + if (OB_SUCC(ret)) { + ObObj mv_mode_default; + mv_mode_default.set_int(0); + ADD_COLUMN_SCHEMA_T("mv_mode", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + mv_mode_default, + mv_mode_default); //default_value + } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); @@ -8468,6 +8487,25 @@ int ObInnerTableSchema::all_virtual_table_history_schema(ObTableSchema &table_sc micro_index_clustered_default, micro_index_clustered_default); //default_value } + + if (OB_SUCC(ret)) { + ObObj mv_mode_default; + mv_mode_default.set_int(0); + ADD_COLUMN_SCHEMA_T("mv_mode", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false, //is_autoincrement + mv_mode_default, + mv_mode_default); //default_value + } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp b/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp index def2bce786..178c3253d2 100644 --- a/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp @@ -9916,6 +9916,63 @@ int ObInnerTableSchema::all_virtual_ls_info_schema(ObTableSchema &table_schema) required_data_disk_size_default, required_data_disk_size_default); //default_value } + + if (OB_SUCC(ret)) { + ObObj mv_major_merge_scn_default; + mv_major_merge_scn_default.set_uint64(0); + ADD_COLUMN_SCHEMA_T("mv_major_merge_scn", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + mv_major_merge_scn_default, + mv_major_merge_scn_default); //default_value + } + + if (OB_SUCC(ret)) { + ObObj mv_publish_scn_default; + mv_publish_scn_default.set_uint64(0); + ADD_COLUMN_SCHEMA_T("mv_publish_scn", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + mv_publish_scn_default, + mv_publish_scn_default); //default_value + } + + if (OB_SUCC(ret)) { + ObObj mv_safe_scn_default; + mv_safe_scn_default.set_uint64(0); + ADD_COLUMN_SCHEMA_T("mv_safe_scn", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + mv_safe_scn_default, + mv_safe_scn_default); //default_value + } if (OB_SUCC(ret)) { table_schema.get_part_option().set_part_num(1); table_schema.set_part_level(PARTITION_LEVEL_ONE); diff --git a/src/share/inner_table/ob_inner_table_schema.15101_15150.cpp b/src/share/inner_table/ob_inner_table_schema.15101_15150.cpp index 99152d1c77..342d623e68 100644 --- a/src/share/inner_table/ob_inner_table_schema.15101_15150.cpp +++ b/src/share/inner_table/ob_inner_table_schema.15101_15150.cpp @@ -6765,6 +6765,21 @@ int ObInnerTableSchema::all_virtual_table_real_agent_ora_schema(ObTableSchema &t false); //is_autoincrement } + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("MV_MODE", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + if (OB_SUCC(ret)) { ADD_COLUMN_SCHEMA("GMT_CREATE", //column_name ++column_id, //column_id diff --git a/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp b/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp index 2caecd2331..83ce512a08 100644 --- a/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp +++ b/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp @@ -6678,6 +6678,21 @@ int ObInnerTableSchema::all_virtual_core_all_table_ora_schema(ObTableSchema &tab false, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("MV_MODE", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } table_schema.set_index_using_type(USING_HASH); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema.15401_15450.cpp b/src/share/inner_table/ob_inner_table_schema.15401_15450.cpp index 6b3b80a637..fc91350a40 100644 --- a/src/share/inner_table/ob_inner_table_schema.15401_15450.cpp +++ b/src/share/inner_table/ob_inner_table_schema.15401_15450.cpp @@ -5108,6 +5108,51 @@ int ObInnerTableSchema::all_virtual_ls_info_ora_schema(ObTableSchema &table_sche false, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("MV_MAJOR_MERGE_SCN", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("MV_PUBLISH_SCN", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("MV_SAFE_SCN", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } if (OB_SUCC(ret)) { table_schema.get_part_option().set_part_num(1); table_schema.set_part_level(PARTITION_LEVEL_ONE); diff --git a/src/share/inner_table/ob_inner_table_schema.1_50.cpp b/src/share/inner_table/ob_inner_table_schema.1_50.cpp index f203688eea..5d30de2d32 100644 --- a/src/share/inner_table/ob_inner_table_schema.1_50.cpp +++ b/src/share/inner_table/ob_inner_table_schema.1_50.cpp @@ -1732,6 +1732,25 @@ int ObInnerTableSchema::all_table_schema(ObTableSchema &table_schema) micro_index_clustered_default, micro_index_clustered_default); //default_value } + + if (OB_SUCC(ret)) { + ObObj mv_mode_default; + mv_mode_default.set_int(0); + ADD_COLUMN_SCHEMA_T("mv_mode", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + mv_mode_default, + mv_mode_default); //default_value + } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp b/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp index 85e1735317..e2872c456c 100644 --- a/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21301_21350.cpp @@ -1036,7 +1036,7 @@ int ObInnerTableSchema::cdb_ob_table_locations_schema(ObTableSchema &table_schem table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT A.TENANT_ID, D.DATABASE_NAME, A.TABLE_NAME, A.TABLE_ID, CASE WHEN A.TABLE_TYPE IN (0) THEN 'SYSTEM TABLE' WHEN A.TABLE_TYPE IN (3,6,8,9) THEN 'USER TABLE' WHEN A.TABLE_TYPE IN (5) THEN 'INDEX' WHEN A.TABLE_TYPE IN (12,13) THEN 'LOB AUX TABLE' WHEN A.TABLE_TYPE IN (15) THEN 'MATERIALIZED VIEW LOG' ELSE NULL END AS TABLE_TYPE, A.PARTITION_NAME, A.SUBPARTITION_NAME, /* INDEX_NAME is valid when table is index */ CASE WHEN A.TABLE_TYPE != 5 THEN NULL WHEN D.DATABASE_NAME != '__recyclebin' THEN SUBSTR(TABLE_NAME, 7 + INSTR(SUBSTR(TABLE_NAME, 7), '_')) ELSE TABLE_NAME END AS INDEX_NAME, CASE WHEN DATA_TABLE_ID = 0 THEN NULL ELSE DATA_TABLE_ID END AS DATA_TABLE_ID, A.TABLET_ID, C.LS_ID, C.ZONE, C.SVR_IP AS SVR_IP, C.SVR_PORT AS SVR_PORT, C.ROLE, C.REPLICA_TYPE, CASE WHEN A.DUPLICATE_SCOPE = 1 THEN 'CLUSTER' ELSE 'NONE' END AS DUPLICATE_SCOPE, CASE WHEN A.DUPLICATE_SCOPE = 1 AND A.DUPLICATE_READ_CONSISTENCY = 0 THEN 'STRONG' WHEN A.DUPLICATE_SCOPE = 1 AND A.DUPLICATE_READ_CONSISTENCY = 1 THEN 'WEAK' ELSE 'NONE' END AS DUPLICATE_READ_CONSISTENCY, A.OBJECT_ID, TG.TABLEGROUP_NAME, TG.TABLEGROUP_ID, TG.SHARDING FROM ( SELECT TENANT_ID, DATABASE_ID, TABLE_NAME, TABLE_ID, 'NULL' AS PARTITION_NAME, 'NULL' AS SUBPARTITION_NAME, TABLE_ID AS OBJECT_ID, TABLET_ID AS TABLET_ID, TABLE_TYPE, DATA_TABLE_ID, DUPLICATE_SCOPE, DUPLICATE_READ_CONSISTENCY, TABLEGROUP_ID FROM OCEANBASE.__ALL_VIRTUAL_CORE_ALL_TABLE WHERE TABLET_ID != 0 UNION ALL SELECT TENANT_ID, DATABASE_ID, TABLE_NAME, TABLE_ID, 'NULL' AS PARTITION_NAME, 'NULL' AS SUBPARTITION_NAME, TABLE_ID AS OBJECT_ID, TABLET_ID AS TABLET_ID, TABLE_TYPE, DATA_TABLE_ID, DUPLICATE_SCOPE, DUPLICATE_READ_CONSISTENCY, TABLEGROUP_ID FROM OCEANBASE.__ALL_VIRTUAL_TABLE WHERE TABLET_ID != 0 AND PART_LEVEL = 0 UNION ALL SELECT P.TENANT_ID AS TENANT_ID, T.DATABASE_ID AS DATABASE_ID, T.TABLE_NAME AS TABLE_NAME, T.TABLE_ID AS TABLE_ID, P.PART_NAME AS PARTITION_NAME, 'NULL' AS SUBPARTITION_NAME, P.PART_ID AS OBJECT_ID, P.TABLET_ID AS TABLET_ID, TABLE_TYPE, DATA_TABLE_ID, DUPLICATE_SCOPE, DUPLICATE_READ_CONSISTENCY, TABLEGROUP_ID FROM OCEANBASE.__ALL_VIRTUAL_TABLE T JOIN OCEANBASE.__ALL_VIRTUAL_PART P ON T.TABLE_ID = P.TABLE_ID WHERE T.TENANT_ID = P.TENANT_ID AND T.PART_LEVEL = 1 AND P.PARTITION_TYPE = 0 UNION ALL SELECT T.TENANT_ID AS TENANT_ID, T.DATABASE_ID AS DATABASE_ID, T.TABLE_NAME AS TABLE_NAME, T.TABLE_ID AS TABLE_ID, P.PART_NAME AS PARTITION_NAME, Q.SUB_PART_NAME AS SUBPARTITION_NAME, Q.SUB_PART_ID AS OBJECT_ID, Q.TABLET_ID AS TABLET_ID, TABLE_TYPE, DATA_TABLE_ID, DUPLICATE_SCOPE, DUPLICATE_READ_CONSISTENCY, TABLEGROUP_ID FROM OCEANBASE.__ALL_VIRTUAL_TABLE T, OCEANBASE.__ALL_VIRTUAL_PART P,OCEANBASE.__ALL_VIRTUAL_SUB_PART Q WHERE T.TABLE_ID =P.TABLE_ID AND P.TABLE_ID=Q.TABLE_ID AND P.PART_ID =Q.PART_ID AND T.TENANT_ID = P.TENANT_ID AND P.TENANT_ID = Q.TENANT_ID AND T.PART_LEVEL = 2 AND P.PARTITION_TYPE = 0 AND Q.PARTITION_TYPE = 0 ) A JOIN OCEANBASE.CDB_OB_TABLET_TO_LS B ON A.TABLET_ID = B.TABLET_ID AND A.TENANT_ID = B.TENANT_ID JOIN OCEANBASE.CDB_OB_LS_LOCATIONS C ON B.LS_ID = C.LS_ID AND A.TENANT_ID = C.TENANT_ID JOIN OCEANBASE.__ALL_VIRTUAL_DATABASE D ON A.TENANT_ID = D.TENANT_ID AND A.DATABASE_ID = D.DATABASE_ID LEFT JOIN OCEANBASE.__ALL_VIRTUAL_TABLEGROUP TG ON A.TABLEGROUP_ID = TG.TABLEGROUP_ID AND A.TENANT_ID = TG.TENANT_ID ORDER BY A.TENANT_ID, A.TABLE_ID, A.TABLET_ID, C.ZONE, SVR_IP, SVR_PORT )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT A.TENANT_ID, D.DATABASE_NAME, A.TABLE_NAME, A.TABLE_ID, CASE WHEN A.TABLE_TYPE IN (0) THEN 'SYSTEM TABLE' WHEN A.TABLE_TYPE IN (3,6,8,9) THEN 'USER TABLE' WHEN A.TABLE_TYPE IN (5) THEN 'INDEX' WHEN A.TABLE_TYPE IN (12,13) THEN 'LOB AUX TABLE' WHEN A.TABLE_TYPE IN (15) THEN 'MATERIALIZED VIEW LOG' ELSE NULL END AS TABLE_TYPE, A.PARTITION_NAME, A.SUBPARTITION_NAME, /* INDEX_NAME is valid when table is index */ CASE WHEN A.TABLE_TYPE != 5 THEN NULL WHEN D.DATABASE_NAME != '__recyclebin' THEN SUBSTR(TABLE_NAME, 7 + INSTR(SUBSTR(TABLE_NAME, 7), '_')) ELSE TABLE_NAME END AS INDEX_NAME, CASE WHEN DATA_TABLE_ID = 0 THEN NULL ELSE DATA_TABLE_ID END AS DATA_TABLE_ID, A.TABLET_ID, C.LS_ID, C.ZONE, C.SVR_IP AS SVR_IP, C.SVR_PORT AS SVR_PORT, C.ROLE, C.REPLICA_TYPE, CASE WHEN A.DUPLICATE_SCOPE = 1 THEN 'CLUSTER' ELSE 'NONE' END AS DUPLICATE_SCOPE, CASE WHEN A.DUPLICATE_SCOPE = 1 AND A.DUPLICATE_READ_CONSISTENCY = 0 THEN 'STRONG' WHEN A.DUPLICATE_SCOPE = 1 AND A.DUPLICATE_READ_CONSISTENCY = 1 THEN 'WEAK' ELSE 'NONE' END AS DUPLICATE_READ_CONSISTENCY, A.OBJECT_ID, TG.TABLEGROUP_NAME, TG.TABLEGROUP_ID, TG.SHARDING FROM ( SELECT TENANT_ID, DATABASE_ID, TABLE_NAME, TABLE_ID, 'NULL' AS PARTITION_NAME, 'NULL' AS SUBPARTITION_NAME, TABLE_ID AS OBJECT_ID, TABLET_ID AS TABLET_ID, TABLE_TYPE, DATA_TABLE_ID, DUPLICATE_SCOPE, DUPLICATE_READ_CONSISTENCY, TABLEGROUP_ID FROM OCEANBASE.__ALL_VIRTUAL_CORE_ALL_TABLE WHERE TABLET_ID != 0 UNION ALL SELECT TENANT_ID, DATABASE_ID, TABLE_NAME, TABLE_ID, 'NULL' AS PARTITION_NAME, 'NULL' AS SUBPARTITION_NAME, TABLE_ID AS OBJECT_ID, TABLET_ID AS TABLET_ID, TABLE_TYPE, DATA_TABLE_ID, DUPLICATE_SCOPE, DUPLICATE_READ_CONSISTENCY, TABLEGROUP_ID FROM OCEANBASE.__ALL_VIRTUAL_TABLE WHERE TABLET_ID != 0 AND PART_LEVEL = 0 UNION ALL SELECT P.TENANT_ID AS TENANT_ID, T.DATABASE_ID AS DATABASE_ID, T.TABLE_NAME AS TABLE_NAME, T.TABLE_ID AS TABLE_ID, P.PART_NAME AS PARTITION_NAME, 'NULL' AS SUBPARTITION_NAME, P.PART_ID AS OBJECT_ID, P.TABLET_ID AS TABLET_ID, TABLE_TYPE, DATA_TABLE_ID, DUPLICATE_SCOPE, DUPLICATE_READ_CONSISTENCY, TABLEGROUP_ID FROM OCEANBASE.__ALL_VIRTUAL_TABLE T JOIN OCEANBASE.__ALL_VIRTUAL_PART P ON T.TABLE_ID = P.TABLE_ID WHERE T.TENANT_ID = P.TENANT_ID AND T.PART_LEVEL = 1 AND P.PARTITION_TYPE = 0 UNION ALL SELECT T.TENANT_ID AS TENANT_ID, T.DATABASE_ID AS DATABASE_ID, T.TABLE_NAME AS TABLE_NAME, T.TABLE_ID AS TABLE_ID, P.PART_NAME AS PARTITION_NAME, Q.SUB_PART_NAME AS SUBPARTITION_NAME, Q.SUB_PART_ID AS OBJECT_ID, Q.TABLET_ID AS TABLET_ID, TABLE_TYPE, DATA_TABLE_ID, DUPLICATE_SCOPE, DUPLICATE_READ_CONSISTENCY, TABLEGROUP_ID FROM OCEANBASE.__ALL_VIRTUAL_TABLE T, OCEANBASE.__ALL_VIRTUAL_PART P,OCEANBASE.__ALL_VIRTUAL_SUB_PART Q WHERE T.TABLE_ID =P.TABLE_ID AND P.TABLE_ID=Q.TABLE_ID AND P.PART_ID =Q.PART_ID AND T.TENANT_ID = P.TENANT_ID AND P.TENANT_ID = Q.TENANT_ID AND T.PART_LEVEL = 2 AND P.PARTITION_TYPE = 0 AND Q.PARTITION_TYPE = 0 ) A JOIN OCEANBASE.CDB_OB_TABLET_TO_LS B ON A.TABLET_ID = B.TABLET_ID AND A.TENANT_ID = B.TENANT_ID JOIN OCEANBASE.CDB_OB_LS_LOCATIONS C ON B.LS_ID = C.LS_ID AND A.TENANT_ID = C.TENANT_ID JOIN OCEANBASE.__ALL_VIRTUAL_DATABASE D ON A.TENANT_ID = D.TENANT_ID AND A.DATABASE_ID = D.DATABASE_ID LEFT JOIN OCEANBASE.__ALL_VIRTUAL_TABLEGROUP TG ON A.TABLEGROUP_ID = TG.TABLEGROUP_ID AND A.TENANT_ID = TG.TENANT_ID ORDER BY A.TENANT_ID, A.TABLE_ID, A.TABLET_ID, C.ZONE, SVR_IP, SVR_PORT )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp b/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp index 8fac8e9d68..680640019a 100644 --- a/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp @@ -1182,7 +1182,7 @@ int ObInnerTableSchema::cdb_mviews_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT B.TENANT_ID AS TENANT_ID, CAST(A.DATABASE_NAME AS CHAR(128)) AS OWNER, CAST(B.TABLE_NAME AS CHAR(128)) AS MVIEW_NAME, CAST(B.TABLE_NAME AS CHAR(128)) AS CONTAINER_NAME, B.VIEW_DEFINITION AS QUERY, CAST(LENGTH(B.VIEW_DEFINITION) AS SIGNED) AS QUERY_LEN, CAST('N' AS CHAR(1)) AS UPDATABLE, CAST(NULL AS CHAR(128)) AS UPDATE_LOG, CAST(NULL AS CHAR(128)) AS MASTER_ROLLBACK_SEG, CAST(NULL AS CHAR(128)) AS MASTER_LINK, CAST( CASE ((B.TABLE_MODE >> 27) & 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS REWRITE_ENABLED, CAST(NULL AS CHAR(9)) AS REWRITE_CAPABILITY, CAST( CASE C.REFRESH_MODE WHEN 0 THEN 'NEVER' WHEN 1 THEN 'DEMAND' WHEN 2 THEN 'COMMIT' WHEN 3 THEN 'STATEMENT' ELSE NULL END AS CHAR(6) ) AS REFRESH_MODE, CAST( CASE C.REFRESH_METHOD WHEN 0 THEN 'NEVER' WHEN 1 THEN 'COMPLETE' WHEN 2 THEN 'FAST' WHEN 3 THEN 'FORCE' ELSE NULL END AS CHAR(8) ) AS REFRESH_METHOD, CAST( CASE C.BUILD_MODE WHEN 0 THEN 'IMMEDIATE' WHEN 1 THEN 'DEFERRED' WHEN 2 THEN 'PERBUILT' ELSE NULL END AS CHAR(9) ) AS BUILD_MODE, CAST(NULL AS CHAR(18)) AS FAST_REFRESHABLE, CAST( CASE C.LAST_REFRESH_TYPE WHEN 0 THEN 'COMPLETE' WHEN 1 THEN 'FAST' ELSE 'NA' END AS CHAR(8) ) AS LAST_REFRESH_TYPE, CAST(C.LAST_REFRESH_DATE AS DATETIME) AS LAST_REFRESH_DATE, CAST(DATE_ADD(C.LAST_REFRESH_DATE, INTERVAL C.LAST_REFRESH_TIME SECOND) AS DATETIME) AS LAST_REFRESH_END_TIME, CAST(NULL AS CHAR(19)) AS STALENESS, CAST(NULL AS CHAR(19)) AS AFTER_FAST_REFRESH, CAST(IF(C.BUILD_MODE = 2, 'Y', 'N') AS CHAR(1)) AS UNKNOWN_PREBUILT, CAST('N' AS CHAR(1)) AS UNKNOWN_PLSQL_FUNC, CAST('N' AS CHAR(1)) AS UNKNOWN_EXTERNAL_TABLE, CAST('N' AS CHAR(1)) AS UNKNOWN_CONSIDER_FRESH, CAST('N' AS CHAR(1)) AS UNKNOWN_IMPORT, CAST('N' AS CHAR(1)) AS UNKNOWN_TRUSTED_FD, CAST(NULL AS CHAR(19)) AS COMPILE_STATE, CAST('Y' AS CHAR(1)) AS USE_NO_INDEX, CAST(NULL AS DATETIME) AS STALE_SINCE, CAST(NULL AS SIGNED) AS NUM_PCT_TABLES, CAST(NULL AS SIGNED) AS NUM_FRESH_PCT_REGIONS, CAST(NULL AS SIGNED) AS NUM_STALE_PCT_REGIONS, CAST('NO' AS CHAR(3)) AS SEGMENT_CREATED, CAST(NULL AS CHAR(128)) AS EVALUATION_EDITION, CAST(NULL AS CHAR(128)) AS UNUSABLE_BEFORE, CAST(NULL AS CHAR(128)) AS UNUSABLE_BEGINNING, CAST(NULL AS CHAR(100)) AS DEFAULT_COLLATION, CAST( CASE ((B.TABLE_MODE >> 28) & 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS ON_QUERY_COMPUTATION FROM oceanbase.__all_virtual_database A, oceanbase.__all_virtual_table B, oceanbase.__all_virtual_mview C WHERE A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = C.TENANT_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT B.TENANT_ID AS TENANT_ID, CAST(A.DATABASE_NAME AS CHAR(128)) AS OWNER, CAST(B.TABLE_NAME AS CHAR(128)) AS MVIEW_NAME, CAST(B.TABLE_NAME AS CHAR(128)) AS CONTAINER_NAME, B.VIEW_DEFINITION AS QUERY, CAST(LENGTH(B.VIEW_DEFINITION) AS SIGNED) AS QUERY_LEN, CAST('N' AS CHAR(1)) AS UPDATABLE, CAST(NULL AS CHAR(128)) AS UPDATE_LOG, CAST(NULL AS CHAR(128)) AS MASTER_ROLLBACK_SEG, CAST(NULL AS CHAR(128)) AS MASTER_LINK, CAST( CASE ((B.TABLE_MODE >> 27) & 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS REWRITE_ENABLED, CAST(NULL AS CHAR(9)) AS REWRITE_CAPABILITY, CAST( CASE C.REFRESH_MODE WHEN 0 THEN 'NEVER' WHEN 1 THEN 'DEMAND' WHEN 2 THEN 'COMMIT' WHEN 3 THEN 'STATEMENT' WHEN 4 THEN 'MAJOR_COMPACTION' ELSE NULL END AS CHAR(32) ) AS REFRESH_MODE, CAST( CASE C.REFRESH_METHOD WHEN 0 THEN 'NEVER' WHEN 1 THEN 'COMPLETE' WHEN 2 THEN 'FAST' WHEN 3 THEN 'FORCE' ELSE NULL END AS CHAR(8) ) AS REFRESH_METHOD, CAST( CASE C.BUILD_MODE WHEN 0 THEN 'IMMEDIATE' WHEN 1 THEN 'DEFERRED' WHEN 2 THEN 'PERBUILT' ELSE NULL END AS CHAR(9) ) AS BUILD_MODE, CAST(NULL AS CHAR(18)) AS FAST_REFRESHABLE, CAST( CASE C.LAST_REFRESH_TYPE WHEN 0 THEN 'COMPLETE' WHEN 1 THEN 'FAST' ELSE 'NA' END AS CHAR(8) ) AS LAST_REFRESH_TYPE, CAST(C.LAST_REFRESH_DATE AS DATETIME) AS LAST_REFRESH_DATE, CAST(DATE_ADD(C.LAST_REFRESH_DATE, INTERVAL C.LAST_REFRESH_TIME SECOND) AS DATETIME) AS LAST_REFRESH_END_TIME, CAST(NULL AS CHAR(19)) AS STALENESS, CAST(NULL AS CHAR(19)) AS AFTER_FAST_REFRESH, CAST(IF(C.BUILD_MODE = 2, 'Y', 'N') AS CHAR(1)) AS UNKNOWN_PREBUILT, CAST('N' AS CHAR(1)) AS UNKNOWN_PLSQL_FUNC, CAST('N' AS CHAR(1)) AS UNKNOWN_EXTERNAL_TABLE, CAST('N' AS CHAR(1)) AS UNKNOWN_CONSIDER_FRESH, CAST('N' AS CHAR(1)) AS UNKNOWN_IMPORT, CAST('N' AS CHAR(1)) AS UNKNOWN_TRUSTED_FD, CAST(NULL AS CHAR(19)) AS COMPILE_STATE, CAST('Y' AS CHAR(1)) AS USE_NO_INDEX, CAST(NULL AS DATETIME) AS STALE_SINCE, CAST(NULL AS SIGNED) AS NUM_PCT_TABLES, CAST(NULL AS SIGNED) AS NUM_FRESH_PCT_REGIONS, CAST(NULL AS SIGNED) AS NUM_STALE_PCT_REGIONS, CAST('NO' AS CHAR(3)) AS SEGMENT_CREATED, CAST(NULL AS CHAR(128)) AS EVALUATION_EDITION, CAST(NULL AS CHAR(128)) AS UNUSABLE_BEFORE, CAST(NULL AS CHAR(128)) AS UNUSABLE_BEGINNING, CAST(NULL AS CHAR(100)) AS DEFAULT_COLLATION, CAST( CASE ((B.TABLE_MODE >> 28) & 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS ON_QUERY_COMPUTATION FROM oceanbase.__all_virtual_database A, oceanbase.__all_virtual_table B, oceanbase.__all_virtual_mview C WHERE A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = C.TENANT_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1233,7 +1233,7 @@ int ObInnerTableSchema::dba_mviews_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS CHAR(128)) AS OWNER, CAST(B.TABLE_NAME AS CHAR(128)) AS MVIEW_NAME, CAST(B.TABLE_NAME AS CHAR(128)) AS CONTAINER_NAME, B.VIEW_DEFINITION AS QUERY, CAST(LENGTH(B.VIEW_DEFINITION) AS SIGNED) AS QUERY_LEN, CAST('N' AS CHAR(1)) AS UPDATABLE, CAST(NULL AS CHAR(128)) AS UPDATE_LOG, CAST(NULL AS CHAR(128)) AS MASTER_ROLLBACK_SEG, CAST(NULL AS CHAR(128)) AS MASTER_LINK, CAST( CASE ((B.TABLE_MODE >> 27) & 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS REWRITE_ENABLED, CAST(NULL AS CHAR(9)) AS REWRITE_CAPABILITY, CAST( CASE C.REFRESH_MODE WHEN 0 THEN 'NEVER' WHEN 1 THEN 'DEMAND' WHEN 2 THEN 'COMMIT' WHEN 3 THEN 'STATEMENT' ELSE NULL END AS CHAR(6) ) AS REFRESH_MODE, CAST( CASE C.REFRESH_METHOD WHEN 0 THEN 'NEVER' WHEN 1 THEN 'COMPLETE' WHEN 2 THEN 'FAST' WHEN 3 THEN 'FORCE' ELSE NULL END AS CHAR(8) ) AS REFRESH_METHOD, CAST( CASE C.BUILD_MODE WHEN 0 THEN 'IMMEDIATE' WHEN 1 THEN 'DEFERRED' WHEN 2 THEN 'PERBUILT' ELSE NULL END AS CHAR(9) ) AS BUILD_MODE, CAST(NULL AS CHAR(18)) AS FAST_REFRESHABLE, CAST( CASE C.LAST_REFRESH_TYPE WHEN 0 THEN 'COMPLETE' WHEN 1 THEN 'FAST' ELSE 'NA' END AS CHAR(8) ) AS LAST_REFRESH_TYPE, CAST(C.LAST_REFRESH_DATE AS DATETIME) AS LAST_REFRESH_DATE, CAST(DATE_ADD(C.LAST_REFRESH_DATE, INTERVAL C.LAST_REFRESH_TIME SECOND) AS DATETIME) AS LAST_REFRESH_END_TIME, CAST(NULL AS CHAR(19)) AS STALENESS, CAST(NULL AS CHAR(19)) AS AFTER_FAST_REFRESH, CAST(IF(C.BUILD_MODE = 2, 'Y', 'N') AS CHAR(1)) AS UNKNOWN_PREBUILT, CAST('N' AS CHAR(1)) AS UNKNOWN_PLSQL_FUNC, CAST('N' AS CHAR(1)) AS UNKNOWN_EXTERNAL_TABLE, CAST('N' AS CHAR(1)) AS UNKNOWN_CONSIDER_FRESH, CAST('N' AS CHAR(1)) AS UNKNOWN_IMPORT, CAST('N' AS CHAR(1)) AS UNKNOWN_TRUSTED_FD, CAST(NULL AS CHAR(19)) AS COMPILE_STATE, CAST('Y' AS CHAR(1)) AS USE_NO_INDEX, CAST(NULL AS DATETIME) AS STALE_SINCE, CAST(NULL AS SIGNED) AS NUM_PCT_TABLES, CAST(NULL AS SIGNED) AS NUM_FRESH_PCT_REGIONS, CAST(NULL AS SIGNED) AS NUM_STALE_PCT_REGIONS, CAST('NO' AS CHAR(3)) AS SEGMENT_CREATED, CAST(NULL AS CHAR(128)) AS EVALUATION_EDITION, CAST(NULL AS CHAR(128)) AS UNUSABLE_BEFORE, CAST(NULL AS CHAR(128)) AS UNUSABLE_BEGINNING, CAST(NULL AS CHAR(100)) AS DEFAULT_COLLATION, CAST( CASE ((B.TABLE_MODE >> 28) & 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS ON_QUERY_COMPUTATION FROM oceanbase.__all_database A, oceanbase.__all_table B, oceanbase.__all_mview C WHERE A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = C.TENANT_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS CHAR(128)) AS OWNER, CAST(B.TABLE_NAME AS CHAR(128)) AS MVIEW_NAME, CAST(B.TABLE_NAME AS CHAR(128)) AS CONTAINER_NAME, B.VIEW_DEFINITION AS QUERY, CAST(LENGTH(B.VIEW_DEFINITION) AS SIGNED) AS QUERY_LEN, CAST('N' AS CHAR(1)) AS UPDATABLE, CAST(NULL AS CHAR(128)) AS UPDATE_LOG, CAST(NULL AS CHAR(128)) AS MASTER_ROLLBACK_SEG, CAST(NULL AS CHAR(128)) AS MASTER_LINK, CAST( CASE ((B.TABLE_MODE >> 27) & 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS REWRITE_ENABLED, CAST(NULL AS CHAR(9)) AS REWRITE_CAPABILITY, CAST( CASE C.REFRESH_MODE WHEN 0 THEN 'NEVER' WHEN 1 THEN 'DEMAND' WHEN 2 THEN 'COMMIT' WHEN 3 THEN 'STATEMENT' WHEN 4 THEN 'MAJOR_COMPACTION' ELSE NULL END AS CHAR(32) ) AS REFRESH_MODE, CAST( CASE C.REFRESH_METHOD WHEN 0 THEN 'NEVER' WHEN 1 THEN 'COMPLETE' WHEN 2 THEN 'FAST' WHEN 3 THEN 'FORCE' ELSE NULL END AS CHAR(8) ) AS REFRESH_METHOD, CAST( CASE C.BUILD_MODE WHEN 0 THEN 'IMMEDIATE' WHEN 1 THEN 'DEFERRED' WHEN 2 THEN 'PERBUILT' ELSE NULL END AS CHAR(9) ) AS BUILD_MODE, CAST(NULL AS CHAR(18)) AS FAST_REFRESHABLE, CAST( CASE C.LAST_REFRESH_TYPE WHEN 0 THEN 'COMPLETE' WHEN 1 THEN 'FAST' ELSE 'NA' END AS CHAR(8) ) AS LAST_REFRESH_TYPE, CAST(C.LAST_REFRESH_DATE AS DATETIME) AS LAST_REFRESH_DATE, CAST(DATE_ADD(C.LAST_REFRESH_DATE, INTERVAL C.LAST_REFRESH_TIME SECOND) AS DATETIME) AS LAST_REFRESH_END_TIME, CAST(NULL AS CHAR(19)) AS STALENESS, CAST(NULL AS CHAR(19)) AS AFTER_FAST_REFRESH, CAST(IF(C.BUILD_MODE = 2, 'Y', 'N') AS CHAR(1)) AS UNKNOWN_PREBUILT, CAST('N' AS CHAR(1)) AS UNKNOWN_PLSQL_FUNC, CAST('N' AS CHAR(1)) AS UNKNOWN_EXTERNAL_TABLE, CAST('N' AS CHAR(1)) AS UNKNOWN_CONSIDER_FRESH, CAST('N' AS CHAR(1)) AS UNKNOWN_IMPORT, CAST('N' AS CHAR(1)) AS UNKNOWN_TRUSTED_FD, CAST(NULL AS CHAR(19)) AS COMPILE_STATE, CAST('Y' AS CHAR(1)) AS USE_NO_INDEX, CAST(NULL AS DATETIME) AS STALE_SINCE, CAST(NULL AS SIGNED) AS NUM_PCT_TABLES, CAST(NULL AS SIGNED) AS NUM_FRESH_PCT_REGIONS, CAST(NULL AS SIGNED) AS NUM_STALE_PCT_REGIONS, CAST('NO' AS CHAR(3)) AS SEGMENT_CREATED, CAST(NULL AS CHAR(128)) AS EVALUATION_EDITION, CAST(NULL AS CHAR(128)) AS UNUSABLE_BEFORE, CAST(NULL AS CHAR(128)) AS UNUSABLE_BEGINNING, CAST(NULL AS CHAR(100)) AS DEFAULT_COLLATION, CAST( CASE ((B.TABLE_MODE >> 28) & 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS ON_QUERY_COMPUTATION FROM oceanbase.__all_database A, oceanbase.__all_table B, oceanbase.__all_mview C WHERE A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = C.TENANT_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.25251_25300.cpp b/src/share/inner_table/ob_inner_table_schema.25251_25300.cpp index 259cf83b7a..6bbee7b2e3 100644 --- a/src/share/inner_table/ob_inner_table_schema.25251_25300.cpp +++ b/src/share/inner_table/ob_inner_table_schema.25251_25300.cpp @@ -1080,7 +1080,7 @@ int ObInnerTableSchema::dba_mviews_ora_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS OWNER, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MVIEW_NAME, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS CONTAINER_NAME, B.VIEW_DEFINITION AS QUERY /* TODO: LONG */, CAST(LENGTH(B.VIEW_DEFINITION) AS NUMBER) AS QUERY_LEN, CAST('N' AS VARCHAR2(1)) AS UPDATABLE, CAST(NULL AS VARCHAR2(128)) AS UPDATE_LOG, CAST(NULL AS VARCHAR2(128)) AS MASTER_ROLLBACK_SEG, CAST(NULL AS VARCHAR2(128)) AS MASTER_LINK, CAST( CASE bitand((B.TABLE_MODE / 134217728), 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS REWRITE_ENABLED, CAST(NULL AS VARCHAR2(9)) AS REWRITE_CAPABILITY, CAST( DECODE(C.REFRESH_MODE, 0, 'NEVER', 1, 'DEMAND', 2, 'COMMIT', 3, 'STATEMENT', NULL ) AS VARCHAR2(6) ) AS REFRESH_MODE, CAST( DECODE(C.REFRESH_METHOD, 0, 'NEVER', 1, 'COMPLETE', 2, 'FAST', 3, 'FORCE', NULL ) AS VARCHAR2(8) ) AS REFRESH_METHOD, CAST( DECODE(C.BUILD_MODE, 0, 'IMMEDIATE', 1, 'DEFERRED', 2, 'PERBUILT', NULL ) AS VARCHAR2(9) ) AS BUILD_MODE, CAST(NULL AS VARCHAR2(18)) AS FAST_REFRESHABLE, CAST( DECODE(C.LAST_REFRESH_TYPE, 0, 'COMPLETE', 1, 'FAST', 'NA' ) AS VARCHAR2(8) ) AS LAST_REFRESH_TYPE, CAST(C.LAST_REFRESH_DATE AS DATE) AS LAST_REFRESH_DATE /* TODO: DD-MON-YYYY */, CAST(C.LAST_REFRESH_DATE + C.LAST_REFRESH_TIME / 86400 AS DATE) AS LAST_REFRESH_END_TIME /* TODO: DD-MON-YYYY */, CAST(NULL AS VARCHAR2(19)) AS STALENESS, CAST(NULL AS VARCHAR2(19)) AS AFTER_FAST_REFRESH, CAST(DECODE(C.BUILD_MODE, 2, 'Y', 'N') AS VARCHAR2(1)) AS UNKNOWN_PREBUILT, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_PLSQL_FUNC, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_EXTERNAL_TABLE, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_CONSIDER_FRESH, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_IMPORT, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_TRUSTED_FD, CAST(NULL AS VARCHAR2(19)) AS COMPILE_STATE, CAST('Y' AS VARCHAR2(1)) AS USE_NO_INDEX, CAST(NULL AS DATE) AS STALE_SINCE, CAST(NULL AS NUMBER) AS NUM_PCT_TABLES, CAST(NULL AS NUMBER) AS NUM_FRESH_PCT_REGIONS, CAST(NULL AS NUMBER) AS NUM_STALE_PCT_REGIONS, CAST('NO' AS VARCHAR2(3)) AS SEGMENT_CREATED, CAST(NULL AS VARCHAR2(128)) AS EVALUATION_EDITION, CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEFORE, CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEGINNING, CAST(NULL AS VARCHAR2(100)) AS DEFAULT_COLLATION, CAST( CASE bitand((B.TABLE_MODE / 268435456), 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS ON_QUERY_COMPUTATION FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, SYS.ALL_VIRTUAL_MVIEW_REAL_AGENT C WHERE A.DATABASE_ID = B.DATABASE_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS OWNER, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MVIEW_NAME, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS CONTAINER_NAME, B.VIEW_DEFINITION AS QUERY /* TODO: LONG */, CAST(LENGTH(B.VIEW_DEFINITION) AS NUMBER) AS QUERY_LEN, CAST('N' AS VARCHAR2(1)) AS UPDATABLE, CAST(NULL AS VARCHAR2(128)) AS UPDATE_LOG, CAST(NULL AS VARCHAR2(128)) AS MASTER_ROLLBACK_SEG, CAST(NULL AS VARCHAR2(128)) AS MASTER_LINK, CAST( CASE bitand((B.TABLE_MODE / 134217728), 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS REWRITE_ENABLED, CAST(NULL AS VARCHAR2(9)) AS REWRITE_CAPABILITY, CAST( DECODE(C.REFRESH_MODE, 0, 'NEVER', 1, 'DEMAND', 2, 'COMMIT', 3, 'STATEMENT', 4, 'MAJOR_COMPACTION', NULL ) AS VARCHAR2(32) ) AS REFRESH_MODE, CAST( DECODE(C.REFRESH_METHOD, 0, 'NEVER', 1, 'COMPLETE', 2, 'FAST', 3, 'FORCE', NULL ) AS VARCHAR2(8) ) AS REFRESH_METHOD, CAST( DECODE(C.BUILD_MODE, 0, 'IMMEDIATE', 1, 'DEFERRED', 2, 'PERBUILT', NULL ) AS VARCHAR2(9) ) AS BUILD_MODE, CAST(NULL AS VARCHAR2(18)) AS FAST_REFRESHABLE, CAST( DECODE(C.LAST_REFRESH_TYPE, 0, 'COMPLETE', 1, 'FAST', 'NA' ) AS VARCHAR2(8) ) AS LAST_REFRESH_TYPE, CAST(C.LAST_REFRESH_DATE AS DATE) AS LAST_REFRESH_DATE /* TODO: DD-MON-YYYY */, CAST(C.LAST_REFRESH_DATE + C.LAST_REFRESH_TIME / 86400 AS DATE) AS LAST_REFRESH_END_TIME /* TODO: DD-MON-YYYY */, CAST(NULL AS VARCHAR2(19)) AS STALENESS, CAST(NULL AS VARCHAR2(19)) AS AFTER_FAST_REFRESH, CAST(DECODE(C.BUILD_MODE, 2, 'Y', 'N') AS VARCHAR2(1)) AS UNKNOWN_PREBUILT, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_PLSQL_FUNC, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_EXTERNAL_TABLE, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_CONSIDER_FRESH, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_IMPORT, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_TRUSTED_FD, CAST(NULL AS VARCHAR2(19)) AS COMPILE_STATE, CAST('Y' AS VARCHAR2(1)) AS USE_NO_INDEX, CAST(NULL AS DATE) AS STALE_SINCE, CAST(NULL AS NUMBER) AS NUM_PCT_TABLES, CAST(NULL AS NUMBER) AS NUM_FRESH_PCT_REGIONS, CAST(NULL AS NUMBER) AS NUM_STALE_PCT_REGIONS, CAST('NO' AS VARCHAR2(3)) AS SEGMENT_CREATED, CAST(NULL AS VARCHAR2(128)) AS EVALUATION_EDITION, CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEFORE, CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEGINNING, CAST(NULL AS VARCHAR2(100)) AS DEFAULT_COLLATION, CAST( CASE bitand((B.TABLE_MODE / 268435456), 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS ON_QUERY_COMPUTATION FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, SYS.ALL_VIRTUAL_MVIEW_REAL_AGENT C WHERE A.DATABASE_ID = B.DATABASE_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1131,7 +1131,7 @@ int ObInnerTableSchema::all_mviews_ora_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS OWNER, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MVIEW_NAME, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS CONTAINER_NAME, B.VIEW_DEFINITION AS QUERY /* TODO: LONG */, CAST(LENGTH(B.VIEW_DEFINITION) AS NUMBER) AS QUERY_LEN, CAST('N' AS VARCHAR2(1)) AS UPDATABLE, CAST(NULL AS VARCHAR2(128)) AS UPDATE_LOG, CAST(NULL AS VARCHAR2(128)) AS MASTER_ROLLBACK_SEG, CAST(NULL AS VARCHAR2(128)) AS MASTER_LINK, CAST( CASE bitand((B.TABLE_MODE / 134217728), 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS REWRITE_ENABLED, CAST(NULL AS VARCHAR2(9)) AS REWRITE_CAPABILITY, CAST( DECODE(C.REFRESH_MODE, 0, 'NEVER', 1, 'DEMAND', 2, 'COMMIT', 3, 'STATEMENT', NULL ) AS VARCHAR2(6) ) AS REFRESH_MODE, CAST( DECODE(C.REFRESH_METHOD, 0, 'NEVER', 1, 'COMPLETE', 2, 'FAST', 3, 'FORCE', NULL ) AS VARCHAR2(8) ) AS REFRESH_METHOD, CAST( DECODE(C.BUILD_MODE, 0, 'IMMEDIATE', 1, 'DEFERRED', 2, 'PERBUILT', NULL ) AS VARCHAR2(9) ) AS BUILD_MODE, CAST(NULL AS VARCHAR2(18)) AS FAST_REFRESHABLE, CAST( DECODE(C.LAST_REFRESH_TYPE, 0, 'COMPLETE', 1, 'FAST', 'NA' ) AS VARCHAR2(8) ) AS LAST_REFRESH_TYPE, CAST(C.LAST_REFRESH_DATE AS DATE) AS LAST_REFRESH_DATE /* TODO: DD-MON-YYYY */, CAST(C.LAST_REFRESH_DATE + C.LAST_REFRESH_TIME / 86400 AS DATE) AS LAST_REFRESH_END_TIME /* TODO: DD-MON-YYYY */, CAST(NULL AS VARCHAR2(19)) AS STALENESS, CAST(NULL AS VARCHAR2(19)) AS AFTER_FAST_REFRESH, CAST(DECODE(C.BUILD_MODE, 2, 'Y', 'N') AS VARCHAR2(1)) AS UNKNOWN_PREBUILT, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_PLSQL_FUNC, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_EXTERNAL_TABLE, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_CONSIDER_FRESH, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_IMPORT, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_TRUSTED_FD, CAST(NULL AS VARCHAR2(19)) AS COMPILE_STATE, CAST('Y' AS VARCHAR2(1)) AS USE_NO_INDEX, CAST(NULL AS DATE) AS STALE_SINCE, CAST(NULL AS NUMBER) AS NUM_PCT_TABLES, CAST(NULL AS NUMBER) AS NUM_FRESH_PCT_REGIONS, CAST(NULL AS NUMBER) AS NUM_STALE_PCT_REGIONS, CAST('NO' AS VARCHAR2(3)) AS SEGMENT_CREATED, CAST(NULL AS VARCHAR2(128)) AS EVALUATION_EDITION, CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEFORE, CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEGINNING, CAST(NULL AS VARCHAR2(100)) AS DEFAULT_COLLATION, CAST( CASE bitand((B.TABLE_MODE / 268435456), 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS ON_QUERY_COMPUTATION FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, SYS.ALL_VIRTUAL_MVIEW_REAL_AGENT C WHERE A.DATABASE_ID = B.DATABASE_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() AND (A.DATABASE_ID = USERENV('SCHEMAID') OR USER_CAN_ACCESS_OBJ(1, B.TABLE_ID, B.DATABASE_ID) = 1) )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS OWNER, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MVIEW_NAME, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS CONTAINER_NAME, B.VIEW_DEFINITION AS QUERY /* TODO: LONG */, CAST(LENGTH(B.VIEW_DEFINITION) AS NUMBER) AS QUERY_LEN, CAST('N' AS VARCHAR2(1)) AS UPDATABLE, CAST(NULL AS VARCHAR2(128)) AS UPDATE_LOG, CAST(NULL AS VARCHAR2(128)) AS MASTER_ROLLBACK_SEG, CAST(NULL AS VARCHAR2(128)) AS MASTER_LINK, CAST( CASE bitand((B.TABLE_MODE / 134217728), 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS REWRITE_ENABLED, CAST(NULL AS VARCHAR2(9)) AS REWRITE_CAPABILITY, CAST( DECODE(C.REFRESH_MODE, 0, 'NEVER', 1, 'DEMAND', 2, 'COMMIT', 3, 'STATEMENT', 4, 'MAJOR_COMPACTION', NULL ) AS VARCHAR2(32) ) AS REFRESH_MODE, CAST( DECODE(C.REFRESH_METHOD, 0, 'NEVER', 1, 'COMPLETE', 2, 'FAST', 3, 'FORCE', NULL ) AS VARCHAR2(8) ) AS REFRESH_METHOD, CAST( DECODE(C.BUILD_MODE, 0, 'IMMEDIATE', 1, 'DEFERRED', 2, 'PERBUILT', NULL ) AS VARCHAR2(9) ) AS BUILD_MODE, CAST(NULL AS VARCHAR2(18)) AS FAST_REFRESHABLE, CAST( DECODE(C.LAST_REFRESH_TYPE, 0, 'COMPLETE', 1, 'FAST', 'NA' ) AS VARCHAR2(8) ) AS LAST_REFRESH_TYPE, CAST(C.LAST_REFRESH_DATE AS DATE) AS LAST_REFRESH_DATE /* TODO: DD-MON-YYYY */, CAST(C.LAST_REFRESH_DATE + C.LAST_REFRESH_TIME / 86400 AS DATE) AS LAST_REFRESH_END_TIME /* TODO: DD-MON-YYYY */, CAST(NULL AS VARCHAR2(19)) AS STALENESS, CAST(NULL AS VARCHAR2(19)) AS AFTER_FAST_REFRESH, CAST(DECODE(C.BUILD_MODE, 2, 'Y', 'N') AS VARCHAR2(1)) AS UNKNOWN_PREBUILT, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_PLSQL_FUNC, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_EXTERNAL_TABLE, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_CONSIDER_FRESH, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_IMPORT, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_TRUSTED_FD, CAST(NULL AS VARCHAR2(19)) AS COMPILE_STATE, CAST('Y' AS VARCHAR2(1)) AS USE_NO_INDEX, CAST(NULL AS DATE) AS STALE_SINCE, CAST(NULL AS NUMBER) AS NUM_PCT_TABLES, CAST(NULL AS NUMBER) AS NUM_FRESH_PCT_REGIONS, CAST(NULL AS NUMBER) AS NUM_STALE_PCT_REGIONS, CAST('NO' AS VARCHAR2(3)) AS SEGMENT_CREATED, CAST(NULL AS VARCHAR2(128)) AS EVALUATION_EDITION, CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEFORE, CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEGINNING, CAST(NULL AS VARCHAR2(100)) AS DEFAULT_COLLATION, CAST( CASE bitand((B.TABLE_MODE / 268435456), 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS ON_QUERY_COMPUTATION FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, SYS.ALL_VIRTUAL_MVIEW_REAL_AGENT C WHERE A.DATABASE_ID = B.DATABASE_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() AND (A.DATABASE_ID = USERENV('SCHEMAID') OR USER_CAN_ACCESS_OBJ(1, B.TABLE_ID, B.DATABASE_ID) = 1) )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1182,7 +1182,7 @@ int ObInnerTableSchema::user_mviews_ora_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS OWNER, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MVIEW_NAME, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS CONTAINER_NAME, B.VIEW_DEFINITION AS QUERY /* TODO: LONG */, CAST(LENGTH(B.VIEW_DEFINITION) AS NUMBER) AS QUERY_LEN, CAST('N' AS VARCHAR2(1)) AS UPDATABLE, CAST(NULL AS VARCHAR2(128)) AS UPDATE_LOG, CAST(NULL AS VARCHAR2(128)) AS MASTER_ROLLBACK_SEG, CAST(NULL AS VARCHAR2(128)) AS MASTER_LINK, CAST( CASE bitand((B.TABLE_MODE / 134217728), 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS REWRITE_ENABLED, CAST(NULL AS VARCHAR2(9)) AS REWRITE_CAPABILITY, CAST( DECODE(C.REFRESH_MODE, 0, 'NEVER', 1, 'DEMAND', 2, 'COMMIT', 3, 'STATEMENT', NULL ) AS VARCHAR2(6) ) AS REFRESH_MODE, CAST( DECODE(C.REFRESH_METHOD, 0, 'NEVER', 1, 'COMPLETE', 2, 'FAST', 3, 'FORCE', NULL ) AS VARCHAR2(8) ) AS REFRESH_METHOD, CAST( DECODE(C.BUILD_MODE, 0, 'IMMEDIATE', 1, 'DEFERRED', 2, 'PERBUILT', NULL ) AS VARCHAR2(9) ) AS BUILD_MODE, CAST(NULL AS VARCHAR2(18)) AS FAST_REFRESHABLE, CAST( DECODE(C.LAST_REFRESH_TYPE, 0, 'COMPLETE', 1, 'FAST', 'NA' ) AS VARCHAR2(8) ) AS LAST_REFRESH_TYPE, CAST(C.LAST_REFRESH_DATE AS DATE) AS LAST_REFRESH_DATE /* TODO: DD-MON-YYYY */, CAST(C.LAST_REFRESH_DATE + C.LAST_REFRESH_TIME / 86400 AS DATE) AS LAST_REFRESH_END_TIME /* TODO: DD-MON-YYYY */, CAST(NULL AS VARCHAR2(19)) AS STALENESS, CAST(NULL AS VARCHAR2(19)) AS AFTER_FAST_REFRESH, CAST(DECODE(C.BUILD_MODE, 2, 'Y', 'N') AS VARCHAR2(1)) AS UNKNOWN_PREBUILT, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_PLSQL_FUNC, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_EXTERNAL_TABLE, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_CONSIDER_FRESH, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_IMPORT, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_TRUSTED_FD, CAST(NULL AS VARCHAR2(19)) AS COMPILE_STATE, CAST('Y' AS VARCHAR2(1)) AS USE_NO_INDEX, CAST(NULL AS DATE) AS STALE_SINCE, CAST(NULL AS NUMBER) AS NUM_PCT_TABLES, CAST(NULL AS NUMBER) AS NUM_FRESH_PCT_REGIONS, CAST(NULL AS NUMBER) AS NUM_STALE_PCT_REGIONS, CAST('NO' AS VARCHAR2(3)) AS SEGMENT_CREATED, CAST(NULL AS VARCHAR2(128)) AS EVALUATION_EDITION, CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEFORE, CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEGINNING, CAST(NULL AS VARCHAR2(100)) AS DEFAULT_COLLATION, CAST( CASE bitand((B.TABLE_MODE / 268435456), 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS ON_QUERY_COMPUTATION FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, SYS.ALL_VIRTUAL_MVIEW_REAL_AGENT C WHERE A.DATABASE_ID = B.DATABASE_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() AND A.DATABASE_NAME = SYS_CONTEXT('USERENV','CURRENT_USER') )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS OWNER, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MVIEW_NAME, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS CONTAINER_NAME, B.VIEW_DEFINITION AS QUERY /* TODO: LONG */, CAST(LENGTH(B.VIEW_DEFINITION) AS NUMBER) AS QUERY_LEN, CAST('N' AS VARCHAR2(1)) AS UPDATABLE, CAST(NULL AS VARCHAR2(128)) AS UPDATE_LOG, CAST(NULL AS VARCHAR2(128)) AS MASTER_ROLLBACK_SEG, CAST(NULL AS VARCHAR2(128)) AS MASTER_LINK, CAST( CASE bitand((B.TABLE_MODE / 134217728), 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS REWRITE_ENABLED, CAST(NULL AS VARCHAR2(9)) AS REWRITE_CAPABILITY, CAST( DECODE(C.REFRESH_MODE, 0, 'NEVER', 1, 'DEMAND', 2, 'COMMIT', 3, 'STATEMENT', 4, 'MAJOR_COMPACTION', NULL ) AS VARCHAR2(32) ) AS REFRESH_MODE, CAST( DECODE(C.REFRESH_METHOD, 0, 'NEVER', 1, 'COMPLETE', 2, 'FAST', 3, 'FORCE', NULL ) AS VARCHAR2(8) ) AS REFRESH_METHOD, CAST( DECODE(C.BUILD_MODE, 0, 'IMMEDIATE', 1, 'DEFERRED', 2, 'PERBUILT', NULL ) AS VARCHAR2(9) ) AS BUILD_MODE, CAST(NULL AS VARCHAR2(18)) AS FAST_REFRESHABLE, CAST( DECODE(C.LAST_REFRESH_TYPE, 0, 'COMPLETE', 1, 'FAST', 'NA' ) AS VARCHAR2(8) ) AS LAST_REFRESH_TYPE, CAST(C.LAST_REFRESH_DATE AS DATE) AS LAST_REFRESH_DATE /* TODO: DD-MON-YYYY */, CAST(C.LAST_REFRESH_DATE + C.LAST_REFRESH_TIME / 86400 AS DATE) AS LAST_REFRESH_END_TIME /* TODO: DD-MON-YYYY */, CAST(NULL AS VARCHAR2(19)) AS STALENESS, CAST(NULL AS VARCHAR2(19)) AS AFTER_FAST_REFRESH, CAST(DECODE(C.BUILD_MODE, 2, 'Y', 'N') AS VARCHAR2(1)) AS UNKNOWN_PREBUILT, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_PLSQL_FUNC, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_EXTERNAL_TABLE, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_CONSIDER_FRESH, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_IMPORT, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_TRUSTED_FD, CAST(NULL AS VARCHAR2(19)) AS COMPILE_STATE, CAST('Y' AS VARCHAR2(1)) AS USE_NO_INDEX, CAST(NULL AS DATE) AS STALE_SINCE, CAST(NULL AS NUMBER) AS NUM_PCT_TABLES, CAST(NULL AS NUMBER) AS NUM_FRESH_PCT_REGIONS, CAST(NULL AS NUMBER) AS NUM_STALE_PCT_REGIONS, CAST('NO' AS VARCHAR2(3)) AS SEGMENT_CREATED, CAST(NULL AS VARCHAR2(128)) AS EVALUATION_EDITION, CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEFORE, CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEGINNING, CAST(NULL AS VARCHAR2(100)) AS DEFAULT_COLLATION, CAST( CASE bitand((B.TABLE_MODE / 268435456), 1) WHEN 0 THEN 'N' WHEN 1 THEN 'Y' ELSE NULL END AS CHAR(1) ) AS ON_QUERY_COMPUTATION FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, SYS.ALL_VIRTUAL_MVIEW_REAL_AGENT C WHERE A.DATABASE_ID = B.DATABASE_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() AND A.DATABASE_NAME = SYS_CONTEXT('USERENV','CURRENT_USER') )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index f7660a72e0..8c83567608 100755 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -319,7 +319,8 @@ all_table_def = dict( ('local_session_vars', 'longtext', 'true'), ('duplicate_read_consistency', 'int', 'false', '0'), ('index_params', 'varchar:OB_MAX_INDEX_PARAMS_LENGTH', 'false', ''), - ('micro_index_clustered', 'bool', 'false', 'false') + ('micro_index_clustered', 'bool', 'false', 'false'), + ('mv_mode', 'int', 'false', '0') ], ) @@ -12582,6 +12583,9 @@ def_table_schema( ('transfer_scn', 'uint'), ('tx_blocked', 'int'), ('required_data_disk_size', 'int', 'false', 0), + ('mv_major_merge_scn', 'uint', 'false', 0), + ('mv_publish_scn', 'uint', 'false', 0), + ('mv_safe_scn', 'uint', 'false', 0), ], partition_columns = ['svr_ip', 'svr_port'], vtable_route_policy = 'distributed', @@ -35163,8 +35167,9 @@ def_table_schema( WHEN 1 THEN 'DEMAND' WHEN 2 THEN 'COMMIT' WHEN 3 THEN 'STATEMENT' + WHEN 4 THEN 'MAJOR_COMPACTION' ELSE NULL - END AS CHAR(6) + END AS CHAR(32) ) AS REFRESH_MODE, CAST( CASE C.REFRESH_METHOD @@ -35265,8 +35270,9 @@ def_table_schema( WHEN 1 THEN 'DEMAND' WHEN 2 THEN 'COMMIT' WHEN 3 THEN 'STATEMENT' + WHEN 4 THEN 'MAJOR_COMPACTION' ELSE NULL - END AS CHAR(6) + END AS CHAR(32) ) AS REFRESH_MODE, CAST( CASE C.REFRESH_METHOD @@ -57518,8 +57524,9 @@ def_table_schema( 1, 'DEMAND', 2, 'COMMIT', 3, 'STATEMENT', + 4, 'MAJOR_COMPACTION', NULL - ) AS VARCHAR2(6) + ) AS VARCHAR2(32) ) AS REFRESH_MODE, CAST( DECODE(C.REFRESH_METHOD, 0, 'NEVER', @@ -57620,8 +57627,9 @@ def_table_schema( 1, 'DEMAND', 2, 'COMMIT', 3, 'STATEMENT', + 4, 'MAJOR_COMPACTION', NULL - ) AS VARCHAR2(6) + ) AS VARCHAR2(32) ) AS REFRESH_MODE, CAST( DECODE(C.REFRESH_METHOD, 0, 'NEVER', @@ -57724,8 +57732,9 @@ def_table_schema( 1, 'DEMAND', 2, 'COMMIT', 3, 'STATEMENT', + 4, 'MAJOR_COMPACTION', NULL - ) AS VARCHAR2(6) + ) AS VARCHAR2(32) ) AS REFRESH_MODE, CAST( DECODE(C.REFRESH_METHOD, 0, 'NEVER', diff --git a/src/share/ls/ob_ls_creator.cpp b/src/share/ls/ob_ls_creator.cpp index b356af6fd4..5591d2208a 100644 --- a/src/share/ls/ob_ls_creator.cpp +++ b/src/share/ls/ob_ls_creator.cpp @@ -34,6 +34,7 @@ #endif #include "share/tenant_snapshot/ob_tenant_snapshot_table_operator.h" #include "share/restore/ob_tenant_clone_table_operator.h" +#include "share/ob_global_stat_proxy.h" // for ObGlobalStatProxy using namespace oceanbase::common; using namespace oceanbase::share; @@ -422,6 +423,47 @@ int ObLSCreator::process_after_has_member_list_( return ret; } +int ObLSCreator::check_tenant_mv_merge_info_(const share::ObAllTenantInfo &tenant_info, + storage::ObMajorMVMergeInfo &major_mv_merge_info) +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(!is_valid() || !tenant_info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_info)); + } else if (tenant_info.is_primary() && id_.is_user_ls()) { + major_mv_merge_info.reset(); + uint64_t data_version; + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id_, data_version))) { + LOG_WARN("fail to get data version", KR(ret), K(tenant_id_)); + } else if (OB_UNLIKELY(data_version < DATA_VERSION_4_3_4_0)) { + LOG_INFO("data version is less than 4.3.4, not get tenant mv merge scn", KR(ret), + K(data_version), K(major_mv_merge_info)); + } else if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret), KP(GCTX.sql_proxy_)); + } else { + ObGlobalStatProxy proxy(*GCTX.sql_proxy_, tenant_id_); + share::SCN tenant_mv_merge_scn; + if (OB_FAIL(proxy.get_major_refresh_mv_merge_scn(true/*for update*/, tenant_mv_merge_scn))) { + if (OB_ERR_NULL_VALUE == ret) { + ret = OB_SUCCESS; + tenant_mv_merge_scn = share::SCN::min_scn(); + } else { + LOG_WARN("fail to get major refresh mv merge scn", KR(ret)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(major_mv_merge_info.init(tenant_mv_merge_scn, tenant_mv_merge_scn, tenant_mv_merge_scn))) { + LOG_WARN("fail to init major mv merge info", KR(ret), K(tenant_mv_merge_scn)); + } + LOG_INFO("create ls with merge info", KR(ret), K(major_mv_merge_info)); + } + } + + return ret; +} + int ObLSCreator::create_ls_(const ObILSAddr &addrs, const int64_t paxos_replica_num, const share::ObAllTenantInfo &tenant_info, @@ -456,18 +498,22 @@ int ObLSCreator::create_ls_(const ObILSAddr &addrs, int tmp_ret = OB_SUCCESS; ObArray return_code_array; const common::ObReplicaProperty replica_property; + storage::ObMajorMVMergeInfo major_mv_merge_info; lib::Worker::CompatMode new_compat_mode = compat_mode == ORACLE_MODE ? lib::Worker::CompatMode::ORACLE : lib::Worker::CompatMode::MYSQL; + if (OB_FAIL(check_tenant_mv_merge_info_(tenant_info, major_mv_merge_info))) { + LOG_WARN("failed to check tenant mv merge info", KR(ret), K(tenant_info), K(major_mv_merge_info)); + } for (int64_t i = 0; OB_SUCC(ret) && i < addrs.count(); ++i) { arg.reset(); const ObLSReplicaAddr &addr = addrs.at(i); if (OB_FAIL(arg.init(tenant_id_, id_, addr.replica_type_, replica_property, tenant_info, create_scn, new_compat_mode, - create_with_palf, palf_base_info))) { + create_with_palf, palf_base_info, major_mv_merge_info))) { LOG_WARN("failed to init create log stream arg", KR(ret), K(addr), K(create_with_palf), K(replica_property), - K_(id), K_(tenant_id), K(tenant_info), K(create_scn), K(new_compat_mode), K(palf_base_info)); + K_(id), K_(tenant_id), K(tenant_info), K(create_scn), K(new_compat_mode), K(palf_base_info), K(major_mv_merge_info)); } else if (OB_TMP_FAIL(create_ls_proxy_.call(addr.addr_, ctx.get_timeout(), GCONF.cluster_id, tenant_id_, arg))) { LOG_WARN("failed to all async rpc", KR(tmp_ret), K(addr), K(ctx.get_timeout()), diff --git a/src/share/ls/ob_ls_creator.h b/src/share/ls/ob_ls_creator.h index 3e227454f2..2694319ff2 100644 --- a/src/share/ls/ob_ls_creator.h +++ b/src/share/ls/ob_ls_creator.h @@ -122,6 +122,8 @@ private: common::ObMemberList &member_list, common::ObMember &arbitration_service, common::GlobalLearnerList &learner_list); + int check_tenant_mv_merge_info_(const share::ObAllTenantInfo &tenant_info, + storage::ObMajorMVMergeInfo &major_mv_merge_info); int check_member_list_and_learner_list_all_in_meta_table_( const common::ObMemberList &member_list, const common::GlobalLearnerList &learner_list); diff --git a/src/share/ls/ob_ls_operator.cpp b/src/share/ls/ob_ls_operator.cpp index c568d7d4bb..8ae8cd3ee1 100644 --- a/src/share/ls/ob_ls_operator.cpp +++ b/src/share/ls/ob_ls_operator.cpp @@ -982,17 +982,50 @@ int ObLSAttrOperator::get_all_ls_by_order(const bool lock_sys_ls, bool only_existing_ls) { int ret = OB_SUCCESS; + ls_operation_array.reset(); ObMySQLTransaction trans; - ObLSAttr sys_ls_attr; - if (OB_UNLIKELY(!is_valid())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("operation is not valid", KR(ret), "operation", *this); } else if (OB_FAIL(trans.start(proxy_, tenant_id_))) { LOG_WARN("failed to start transaction", KR(ret), K_(tenant_id)); - /* to get accurate LS list need lock SYS_LS */ + } else if (OB_FAIL(get_all_ls_by_order_in_trans(lock_sys_ls, + ls_operation_array, + trans, + only_existing_ls))) { + LOG_WARN("failed get all ls in trans", KR(ret), K_(tenant_id)); + } + + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("failed to end trans", KR(ret), KR(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + } + + + return ret; +} + +int ObLSAttrOperator::get_all_ls_by_order_in_trans(const bool lock_sys_ls, + ObLSAttrIArray &ls_operation_array, + common::ObMySQLTransaction &trans, + bool only_existing_ls) +{ + int ret = OB_SUCCESS; + ls_operation_array.reset(); + ObLSAttr sys_ls_attr; + + if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("operation is not valid", KR(ret), "operation", *this); + } else if (!trans.is_started()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("transaction is not started", KR(ret)); } else if (lock_sys_ls && OB_FAIL(get_ls_attr(SYS_LS, true /* for_update */, trans, sys_ls_attr))) { + /* to get accurate LS list need lock SYS_LS */ LOG_WARN("failed to load sys ls status", KR(ret)); } else { ObSqlString sql; @@ -1005,19 +1038,11 @@ int ObLSAttrOperator::get_all_ls_by_order(const bool lock_sys_ls, LOG_WARN("failed to append", KR(ret), K(sql)); } } - if (FAILEDx(exec_read(tenant_id_, sql, *proxy_, this, ls_operation_array))) { + if (FAILEDx(exec_read(tenant_id_, sql, trans, this, ls_operation_array))) { LOG_WARN("failed to construct ls attr", KR(ret), K(sql)); } } - if (trans.is_started()) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { - LOG_WARN("failed to end trans", KR(ret), KR(tmp_ret)); - ret = OB_SUCC(ret) ? tmp_ret : ret; - } - } - return ret; } diff --git a/src/share/ls/ob_ls_operator.h b/src/share/ls/ob_ls_operator.h index af3e31b79f..55ffbf49db 100644 --- a/src/share/ls/ob_ls_operator.h +++ b/src/share/ls/ob_ls_operator.h @@ -325,6 +325,10 @@ public: int get_all_ls_by_order(const bool lock_sys_ls, ObLSAttrIArray &ls_operation_array, bool only_existing_ls = true); + int get_all_ls_by_order_in_trans(const bool lock_sys_ls, + ObLSAttrIArray &ls_operation_array, + common::ObMySQLTransaction &trans, + bool only_existing_ls = true); int insert_ls(const ObLSAttr &ls_attr, const ObTenantSwitchoverStatus &working_sw_status, ObMySQLTransaction *trans = NULL); diff --git a/src/share/ob_balance_define.cpp b/src/share/ob_balance_define.cpp index 34422323a8..d3b4b24945 100644 --- a/src/share/ob_balance_define.cpp +++ b/src/share/ob_balance_define.cpp @@ -31,14 +31,14 @@ bool check_if_need_balance_table( const char *&table_type_str) { bool need_balance = false; - if (table_schema.is_duplicate_table()) { + if (table_schema.is_broadcast_table() || table_schema.is_duplicate_table()) { table_type_str = "DUPLICATE TABLE"; } else if (table_schema.is_index_table() && !table_schema.is_global_index_table()) { table_type_str = "LOCAL INDEX"; } else { table_type_str = ob_table_type_str(table_schema.get_table_type()); } - need_balance = !table_schema.is_duplicate_table() + need_balance = !(table_schema.is_broadcast_table() || table_schema.is_duplicate_table()) && (table_schema.is_user_table() || table_schema.is_global_index_table() || table_schema.is_tmp_table()); diff --git a/src/share/ob_debug_sync_point.h b/src/share/ob_debug_sync_point.h index fc49e59ea3..0777311cb5 100755 --- a/src/share/ob_debug_sync_point.h +++ b/src/share/ob_debug_sync_point.h @@ -641,12 +641,15 @@ class ObString; ACT(BEFORE_RESTORE_CREATE_TABLETS_SSTABLE,)\ ACT(BEFORE_CLOSE_BACKUP_INDEX_BUILDER,)\ ACT(BEFROE_UPDATE_DATA_VERSION,)\ + ACT(BEFORE_TABLET_MIGRATION_DAG_INNER_RETRY,)\ ACT(BEFORE_DBMS_VECTOR_REFRESH,)\ ACT(BEFORE_DBMS_VECTOR_REBUILD,)\ ACT(BEFORE_DATA_DICT_DUMP_FINISH,)\ ACT(AFTER_PHYSICAL_RESTORE_CREATE_TENANT,)\ ACT(BEFROE_UPDATE_MIG_TABLET_CONVERT_CO_PROGRESSING,)\ ACT(AFTER_SET_CO_CONVERT_RETRY_EXHUASTED,)\ + ACT(BEFORE_FOLLOWER_REPLACE_REMOTE_SSTABLE,)\ + ACT(BEFORE_MIGRATION_LS_OFFLINE,)\ ACT(AFTER_CREATE_SPLIT_TASK,)\ ACT(BEFORE_PARTITION_SPLIT_TASK_CLEANUP,)\ ACT(BEFORE_TABLET_SPLIT_PREPARE_TASK,)\ diff --git a/src/share/ob_duplicate_scope_define.h b/src/share/ob_duplicate_scope_define.h index 1395980900..808086aeb1 100644 --- a/src/share/ob_duplicate_scope_define.h +++ b/src/share/ob_duplicate_scope_define.h @@ -85,7 +85,48 @@ public: ret = convert_duplicate_scope_string(this_string, duplicate_scope); return ret; } + static bool is_valid_duplicate_scope(const ObDuplicateScope duplicate_scope) { + return duplicate_scope == ObDuplicateScope::DUPLICATE_SCOPE_CLUSTER; + } }; + +class ObDuplicateReadConsistencyChecker +{ +public: + static bool is_valid_duplicate_read_consistency(ObDuplicateReadConsistency read_consistency) { + return read_consistency < ObDuplicateReadConsistency::MAX + && read_consistency >= ObDuplicateReadConsistency::STRONG; + } + static int convert_duplicate_read_consistency_string( + const common::ObString &duplicate_read_consistency_str, + ObDuplicateReadConsistency &duplicate_read_consistency) { + int ret = OB_SUCCESS; + duplicate_read_consistency = ObDuplicateReadConsistency::MAX; + bool found = false; + for (int64_t idx = static_cast(ObDuplicateReadConsistency::STRONG); + idx < static_cast(ObDuplicateReadConsistency::MAX) && !found; + ++idx) { + if (0 == common::ObString::make_string(duplicate_read_consistency_strings[idx]).case_compare(duplicate_read_consistency_str)) { + found = true; + duplicate_read_consistency = static_cast(idx); + } + } + if (!found) { + ret = OB_INVALID_ARGUMENT; + } + return ret; + } + static const char* get_duplicate_read_consistency_str(const ObDuplicateReadConsistency &duplicate_read_consistency) { + switch (duplicate_read_consistency) { + case ObDuplicateReadConsistency::STRONG: + case ObDuplicateReadConsistency::WEAK: + return duplicate_read_consistency_strings[static_cast(duplicate_read_consistency)]; + default: + return "UNKNOWN"; + } + } +}; + } // share } // oceanbase diff --git a/src/share/ob_errno.cpp b/src/share/ob_errno.cpp index cd6b81e3e6..81d4d728a1 100644 --- a/src/share/ob_errno.cpp +++ b/src/share/ob_errno.cpp @@ -23827,6 +23827,20 @@ static const _error _error_OB_PARTITION_ALREADY_BALANCED = { .ob_str_error = "OBE-00600: internal error code, arguments: -7124, partitions are already balanced", .ob_str_user_error = "OBE-00600: internal error code, arguments: -7124, partitions are already balanced, %s" }; +static const _error _error_NEW_MV_MAJOR_VERSION_NOT_MATCH = { + .error_name = "NEW_MV_MAJOR_VERSION_NOT_MATCH", + .error_cause = "Internal Error", + .error_solution = "Contact OceanBase Support", + .mysql_errno = -1, + .sqlstate = "HY000", + .str_error = "new mv major version is not match", + .str_user_error = "new mv major version is not match", + .oracle_errno = 600, + .oracle_str_error = "ORA-00600: internal error code, arguments: -7125, new mv major version is not match", + .oracle_str_user_error = "ORA-00600: internal error code, arguments: -7125, new mv major version is not match", + .ob_str_error = "OBE-00600: internal error code, arguments: -7125, new mv major version is not match", + .ob_str_user_error = "OBE-00600: internal error code, arguments: -7125, new mv major version is not match" +}; static const _error _error_OB_ERR_GIS_DIFFERENT_SRIDS = { .error_name = "OB_ERR_GIS_DIFFERENT_SRIDS", .error_cause = "Internal Error", @@ -34732,6 +34746,7 @@ struct ObStrErrorInit _errors[-OB_SEQUENCE_TOO_SMALL] = &_error_OB_SEQUENCE_TOO_SMALL; _errors[-OB_TRANSFER_CANNOT_START] = &_error_OB_TRANSFER_CANNOT_START; _errors[-OB_PARTITION_ALREADY_BALANCED] = &_error_OB_PARTITION_ALREADY_BALANCED; + _errors[-NEW_MV_MAJOR_VERSION_NOT_MATCH] = &_error_NEW_MV_MAJOR_VERSION_NOT_MATCH; _errors[-OB_ERR_GIS_DIFFERENT_SRIDS] = &_error_OB_ERR_GIS_DIFFERENT_SRIDS; _errors[-OB_ERR_GIS_UNSUPPORTED_ARGUMENT] = &_error_OB_ERR_GIS_UNSUPPORTED_ARGUMENT; _errors[-OB_ERR_GIS_UNKNOWN_ERROR] = &_error_OB_ERR_GIS_UNKNOWN_ERROR; @@ -35423,7 +35438,7 @@ namespace oceanbase { namespace common { -int g_all_ob_errnos[2356] = {0, -4000, -4001, -4002, -4003, -4004, -4005, -4006, -4007, -4008, -4009, -4010, -4011, -4012, -4013, -4014, -4015, -4016, -4017, -4018, -4019, -4020, -4021, -4022, -4023, -4024, -4025, -4026, -4027, -4028, -4029, -4030, -4031, -4032, -4033, -4034, -4035, -4036, -4037, -4038, -4039, -4041, -4042, -4043, -4044, -4045, -4046, -4047, -4048, -4049, -4050, -4051, -4052, -4053, -4054, -4055, -4057, -4058, -4060, -4061, -4062, -4063, -4064, -4065, -4066, -4067, -4068, -4070, -4071, -4072, -4073, -4074, -4075, -4076, -4077, -4078, -4080, -4081, -4084, -4085, -4090, -4097, -4098, -4099, -4100, -4101, -4102, -4103, -4104, -4105, -4106, -4107, -4108, -4109, -4110, -4111, -4112, -4113, -4114, -4115, -4116, -4117, -4118, -4119, -4120, -4121, -4122, -4123, -4124, -4125, -4126, -4127, -4128, -4133, -4138, -4139, -4142, -4143, -4144, -4146, -4147, -4149, -4150, -4151, -4152, -4153, -4154, -4155, -4156, -4157, -4158, -4159, -4160, -4161, -4162, -4163, -4164, -4165, -4166, -4167, -4168, -4169, -4170, -4171, -4172, -4173, -4174, -4175, -4176, -4177, -4178, -4179, -4180, -4181, -4182, -4183, -4184, -4185, -4186, -4187, -4188, -4189, -4190, -4191, -4192, -4200, -4201, -4204, -4205, -4206, -4207, -4208, -4209, -4210, -4211, -4212, -4213, -4214, -4215, -4216, -4217, -4218, -4219, -4220, -4221, -4222, -4223, -4224, -4225, -4226, -4227, -4228, -4229, -4230, -4231, -4232, -4233, -4234, -4235, -4236, -4237, -4238, -4239, -4240, -4241, -4242, -4243, -4244, -4245, -4246, -4247, -4248, -4249, -4250, -4251, -4252, -4253, -4254, -4255, -4256, -4257, -4258, -4260, -4261, -4262, -4263, -4264, -4265, -4266, -4267, -4268, -4269, -4270, -4271, -4273, -4274, -4275, -4276, -4277, -4278, -4279, -4280, -4281, -4282, -4283, -4284, -4285, -4286, -4287, -4288, -4289, -4290, -4291, -4292, -4293, -4294, -4295, -4296, -4297, -4298, -4299, -4300, -4301, -4302, -4303, -4304, -4305, -4306, -4307, -4308, -4309, -4310, -4311, -4312, -4313, -4314, -4315, -4316, -4317, -4318, -4319, -4320, -4321, -4322, -4323, -4324, -4325, -4326, -4327, -4328, -4329, -4330, -4331, -4332, -4333, -4334, -4335, -4336, -4337, -4338, -4339, -4340, -4341, -4342, -4343, -4344, -4345, -4346, -4347, -4348, -4349, -4350, -4351, -4352, -4353, -4354, -4355, -4356, -4357, -4358, -4359, -4360, -4361, -4362, -4363, -4364, -4365, -4366, -4367, -4368, -4369, -4370, -4371, -4372, -4373, -4374, -4375, -4376, -4377, -4378, -4379, -4380, -4381, -4382, -4383, -4385, -4386, -4387, -4388, -4389, -4390, -4391, -4392, -4393, -4394, -4395, -4396, -4397, -4398, -4399, -4400, -4401, -4402, -4403, -4404, -4505, -4507, -4510, -4512, -4515, -4517, -4518, -4519, -4523, -4524, -4525, -4526, -4527, -4528, -4529, -4530, -4531, -4532, -4533, -4537, -4538, -4539, -4540, -4541, -4542, -4543, -4544, -4545, -4546, -4547, -4548, -4549, -4550, -4551, -4552, -4553, -4554, -4600, -4601, -4602, -4603, -4604, -4605, -4606, -4607, -4608, -4609, -4610, -4611, -4613, -4614, -4615, -4620, -4621, -4622, -4623, -4624, -4625, -4626, -4628, -4629, -4630, -4631, -4632, -4633, -4634, -4636, -4637, -4638, -4639, -4640, -4641, -4642, -4643, -4644, -4645, -4646, -4647, -4648, -4649, -4650, -4651, -4652, -4653, -4654, -4655, -4656, -4657, -4658, -4659, -4660, -4661, -4662, -4663, -4664, -4665, -4666, -4667, -4668, -4669, -4670, -4671, -4672, -4673, -4674, -4675, -4676, -4677, -4678, -4679, -4680, -4681, -4682, -4683, -4684, -4685, -4686, -4687, -4688, -4689, -4690, -4691, -4692, -4693, -4694, -4695, -4696, -4697, -4698, -4699, -4700, -4701, -4702, -4703, -4704, -4705, -4706, -4707, -4708, -4709, -4710, -4711, -4712, -4713, -4714, -4715, -4716, -4717, -4718, -4719, -4720, -4721, -4722, -4723, -4724, -4725, -4726, -4727, -4728, -4729, -4730, -4731, -4732, -4733, -4734, -4735, -4736, -4737, -4738, -4739, -4740, -4741, -4742, -4743, -4744, -4745, -4746, -4747, -4748, -4749, -4750, -4751, -4752, -4753, -4754, -4755, -4756, -4757, -4758, -4759, -4760, -4761, -4762, -4763, -4764, -4765, -4766, -4767, -4768, -4769, -4770, -4771, -4772, -4773, -4774, -4775, -4776, -4777, -4778, -4779, -4780, -4781, -4782, -4783, -4784, -4785, -5000, -5001, -5002, -5003, -5006, -5007, -5008, -5010, -5011, -5012, -5014, -5015, -5016, -5017, -5018, -5019, -5020, -5022, -5023, -5024, -5025, -5026, -5027, -5028, -5029, -5030, -5031, -5032, -5034, -5035, -5036, -5037, -5038, -5039, -5040, -5041, -5042, -5043, -5044, -5046, -5047, -5050, -5051, -5052, -5053, -5054, -5055, -5056, -5057, -5058, -5059, -5061, -5063, -5064, -5065, -5066, -5067, -5068, -5069, -5070, -5071, -5072, -5073, -5074, -5080, -5081, -5083, -5084, -5085, -5086, -5087, -5088, -5089, -5090, -5091, -5092, -5093, -5094, -5095, -5096, -5097, -5098, -5099, -5100, -5101, -5102, -5103, -5104, -5105, -5106, -5107, -5108, -5109, -5110, -5111, -5112, -5113, -5114, -5115, -5116, -5117, -5118, -5119, -5120, -5121, -5122, -5123, -5124, -5125, -5130, -5131, -5133, -5134, -5135, -5136, -5137, -5138, -5139, -5140, -5142, -5143, -5144, -5145, -5146, -5147, -5148, -5149, -5150, -5151, -5153, -5154, -5155, -5156, -5157, -5158, -5159, -5160, -5161, -5162, -5163, -5164, -5165, -5166, -5167, -5168, -5169, -5170, -5171, -5172, -5173, -5174, -5175, -5176, -5177, -5178, -5179, -5180, -5181, -5182, -5183, -5184, -5185, -5187, -5188, -5189, -5190, -5191, -5192, -5193, -5194, -5195, -5196, -5197, -5198, -5199, -5200, -5201, -5202, -5203, -5204, -5205, -5206, -5207, -5208, -5209, -5210, -5211, -5212, -5213, -5214, -5215, -5216, -5217, -5218, -5219, -5220, -5221, -5222, -5223, -5224, -5225, -5226, -5227, -5228, -5229, -5230, -5231, -5233, -5234, -5235, -5236, -5237, -5238, -5239, -5240, -5241, -5242, -5243, -5244, -5245, -5246, -5247, -5248, -5249, -5250, -5251, -5252, -5253, -5254, -5255, -5256, -5257, -5258, -5259, -5260, -5261, -5262, -5263, -5264, -5265, -5266, -5267, -5268, -5269, -5270, -5271, -5272, -5273, -5274, -5275, -5276, -5277, -5278, -5279, -5280, -5281, -5282, -5283, -5284, -5285, -5286, -5287, -5288, -5289, -5290, -5291, -5292, -5293, -5294, -5295, -5296, -5297, -5298, -5299, -5300, -5301, -5302, -5303, -5304, -5305, -5306, -5307, -5308, -5309, -5310, -5311, -5312, -5313, -5314, -5315, -5316, -5317, -5318, -5319, -5320, -5321, -5322, -5323, -5324, -5325, -5326, -5327, -5328, -5329, -5330, -5331, -5332, -5333, -5334, -5335, -5336, -5337, -5338, -5339, -5340, -5341, -5342, -5343, -5344, -5345, -5346, -5347, -5348, -5349, -5350, -5351, -5352, -5353, -5354, -5355, -5356, -5357, -5358, -5359, -5360, -5361, -5362, -5363, -5364, -5365, -5366, -5367, -5368, -5369, -5370, -5371, -5372, -5373, -5374, -5375, -5376, -5377, -5378, -5379, -5380, -5381, -5382, -5383, -5384, -5385, -5386, -5387, -5388, -5389, -5390, -5400, -5401, -5402, -5403, -5404, -5405, -5406, -5407, -5408, -5409, -5410, -5411, -5412, -5413, -5414, -5415, -5416, -5417, -5418, -5419, -5420, -5421, -5422, -5423, -5424, -5425, -5426, -5427, -5428, -5429, -5430, -5431, -5432, -5433, -5434, -5435, -5436, -5437, -5438, -5439, -5440, -5441, -5442, -5443, -5444, -5445, -5446, -5447, -5448, -5449, -5450, -5451, -5452, -5453, -5454, -5455, -5456, -5457, -5458, -5459, -5460, -5461, -5462, -5463, -5464, -5465, -5466, -5467, -5468, -5469, -5470, -5471, -5472, -5473, -5474, -5475, -5476, -5477, -5478, -5479, -5480, -5481, -5482, -5483, -5484, -5485, -5486, -5487, -5488, -5489, -5490, -5491, -5492, -5493, -5494, -5495, -5496, -5497, -5498, -5499, -5500, -5501, -5502, -5503, -5504, -5505, -5506, -5507, -5508, -5509, -5510, -5511, -5512, -5513, -5514, -5515, -5516, -5517, -5518, -5519, -5520, -5521, -5522, -5540, -5541, -5542, -5543, -5544, -5545, -5546, -5547, -5548, -5549, -5550, -5551, -5552, -5553, -5554, -5555, -5556, -5557, -5558, -5559, -5560, -5561, -5562, -5563, -5564, -5565, -5566, -5567, -5568, -5569, -5570, -5571, -5572, -5573, -5574, -5575, -5576, -5577, -5578, -5579, -5580, -5581, -5582, -5583, -5584, -5585, -5586, -5587, -5588, -5589, -5590, -5591, -5592, -5593, -5594, -5595, -5596, -5597, -5598, -5599, -5600, -5601, -5602, -5603, -5604, -5605, -5607, -5608, -5609, -5610, -5611, -5612, -5613, -5614, -5615, -5616, -5617, -5618, -5619, -5620, -5621, -5622, -5623, -5624, -5625, -5626, -5627, -5628, -5629, -5630, -5631, -5632, -5633, -5634, -5635, -5636, -5637, -5638, -5639, -5640, -5641, -5642, -5643, -5644, -5645, -5646, -5647, -5648, -5649, -5650, -5651, -5652, -5653, -5654, -5655, -5656, -5657, -5658, -5659, -5660, -5661, -5662, -5663, -5664, -5665, -5666, -5667, -5668, -5671, -5672, -5673, -5674, -5675, -5676, -5677, -5678, -5679, -5680, -5681, -5682, -5683, -5684, -5685, -5686, -5687, -5688, -5689, -5690, -5691, -5692, -5693, -5694, -5695, -5696, -5697, -5698, -5699, -5700, -5701, -5702, -5703, -5704, -5705, -5706, -5707, -5708, -5709, -5710, -5711, -5712, -5713, -5714, -5715, -5716, -5717, -5718, -5719, -5720, -5721, -5722, -5723, -5724, -5725, -5726, -5727, -5728, -5729, -5730, -5731, -5732, -5733, -5734, -5735, -5736, -5737, -5738, -5739, -5740, -5741, -5742, -5743, -5744, -5745, -5746, -5747, -5748, -5749, -5750, -5751, -5752, -5753, -5754, -5755, -5756, -5757, -5758, -5759, -5760, -5761, -5762, -5763, -5764, -5765, -5766, -5768, -5769, -5770, -5771, -5772, -5773, -5774, -5777, -5778, -5779, -5780, -5781, -5785, -5786, -5787, -5788, -5789, -5790, -5791, -5792, -5793, -5794, -5795, -5796, -5797, -5798, -5799, -5800, -5801, -5802, -5803, -5804, -5805, -5806, -5807, -5808, -5809, -5810, -5811, -5812, -5813, -5814, -5815, -5816, -5817, -5818, -5819, -5820, -5821, -5822, -5823, -5824, -5825, -5826, -5827, -5828, -5829, -5830, -5831, -5832, -5833, -5834, -5835, -5836, -5837, -5838, -5839, -5840, -5841, -5842, -5843, -5844, -5845, -5846, -5847, -5848, -5849, -5850, -5851, -5852, -5853, -5854, -5855, -5856, -5857, -5858, -5859, -5860, -5861, -5862, -5863, -5864, -5865, -5866, -5867, -5868, -5869, -5870, -5871, -5872, -5873, -5874, -5875, -5876, -5877, -5878, -5879, -5880, -5881, -5882, -5883, -5884, -5885, -5886, -5887, -5888, -5889, -5890, -5891, -5892, -5893, -5894, -5895, -5896, -5897, -5898, -5899, -5900, -5901, -5902, -5903, -5904, -5905, -5906, -5907, -5908, -5909, -5910, -5911, -5912, -5913, -5914, -5915, -5916, -5917, -5918, -5919, -5920, -5921, -5922, -5923, -5924, -5925, -5926, -5927, -5928, -5929, -5930, -5931, -5932, -5933, -5934, -5935, -5936, -5937, -5938, -5939, -5940, -5941, -5942, -5943, -5944, -5945, -5946, -5947, -5948, -5949, -5950, -5951, -5952, -5953, -5954, -5955, -5956, -5957, -5958, -5959, -5960, -5961, -5962, -5963, -5964, -5965, -5966, -5967, -5968, -5969, -5970, -5971, -5972, -5973, -5974, -5975, -5976, -5977, -5978, -5979, -5980, -5981, -5982, -5983, -5984, -5985, -5986, -5987, -5988, -5989, -5990, -5991, -5992, -5993, -5994, -5995, -5996, -5997, -5998, -5999, -6000, -6001, -6002, -6003, -6004, -6005, -6006, -6201, -6202, -6203, -6204, -6205, -6206, -6207, -6208, -6209, -6210, -6211, -6212, -6213, -6214, -6215, -6219, -6220, -6221, -6222, -6223, -6224, -6225, -6226, -6227, -6228, -6229, -6230, -6231, -6232, -6233, -6234, -6235, -6236, -6237, -6238, -6239, -6240, -6241, -6242, -6243, -6244, -6245, -6246, -6247, -6248, -6249, -6250, -6251, -6252, -6253, -6254, -6255, -6256, -6257, -6258, -6259, -6260, -6261, -6262, -6263, -6264, -6265, -6266, -6267, -6268, -6269, -6270, -6271, -6272, -6273, -6274, -6275, -6276, -6277, -6278, -6279, -6280, -6281, -6282, -6283, -6284, -6285, -6301, -6302, -6303, -6304, -6305, -6306, -6307, -6308, -6309, -6310, -6311, -6312, -6313, -6314, -6315, -6316, -6317, -6318, -6319, -6320, -6321, -6322, -6323, -6324, -6325, -6326, -6327, -6328, -6329, -6330, -6331, -6332, -7000, -7001, -7002, -7003, -7004, -7005, -7006, -7007, -7010, -7011, -7012, -7013, -7014, -7015, -7021, -7022, -7024, -7025, -7026, -7027, -7029, -7030, -7031, -7032, -7033, -7034, -7035, -7036, -7037, -7038, -7039, -7040, -7041, -7100, -7101, -7102, -7103, -7104, -7105, -7106, -7107, -7108, -7109, -7110, -7111, -7112, -7113, -7114, -7115, -7116, -7117, -7118, -7119, -7120, -7121, -7122, -7123, -7124, -7201, -7202, -7203, -7204, -7205, -7206, -7207, -7208, -7209, -7210, -7211, -7212, -7213, -7214, -7215, -7216, -7217, -7218, -7219, -7220, -7221, -7222, -7223, -7224, -7225, -7226, -7227, -7228, -7229, -7230, -7231, -7232, -7233, -7234, -7235, -7236, -7237, -7238, -7239, -7240, -7241, -7242, -7243, -7244, -7246, -7247, -7248, -7249, -7250, -7251, -7252, -7253, -7254, -7255, -7256, -7257, -7258, -7259, -7260, -7261, -7262, -7263, -7264, -7265, -7266, -7267, -7268, -7269, -7270, -7271, -7272, -7273, -7274, -7275, -7276, -7277, -7278, -7279, -7280, -7281, -7282, -7283, -7284, -7285, -7286, -7287, -7288, -7289, -7290, -7291, -7292, -7293, -7294, -7295, -7296, -7297, -7298, -7299, -7300, -7301, -7302, -7402, -7403, -7404, -7405, -7406, -7407, -7408, -7409, -7410, -7411, -7412, -7413, -7414, -7415, -7416, -7417, -7418, -7419, -7420, -7421, -7422, -7423, -7424, -7425, -7426, -7427, -7428, -7429, -7430, -7431, -7432, -7433, -7434, -7435, -7600, -7601, -7602, -7603, -7604, -8001, -8002, -8003, -8004, -8005, -9001, -9002, -9003, -9004, -9005, -9006, -9007, -9008, -9009, -9010, -9011, -9012, -9013, -9014, -9015, -9016, -9017, -9018, -9019, -9020, -9022, -9023, -9024, -9025, -9026, -9027, -9028, -9029, -9030, -9031, -9032, -9033, -9034, -9035, -9036, -9037, -9038, -9039, -9040, -9041, -9042, -9043, -9044, -9045, -9046, -9047, -9048, -9049, -9050, -9051, -9052, -9053, -9054, -9057, -9058, -9059, -9060, -9061, -9062, -9063, -9064, -9065, -9066, -9069, -9070, -9071, -9072, -9073, -9074, -9075, -9076, -9077, -9078, -9079, -9080, -9081, -9082, -9083, -9084, -9085, -9086, -9087, -9088, -9089, -9090, -9091, -9092, -9093, -9094, -9095, -9096, -9097, -9098, -9099, -9100, -9101, -9102, -9103, -9104, -9105, -9106, -9107, -9108, -9109, -9110, -9111, -9112, -9113, -9114, -9115, -9116, -9117, -9118, -9119, -9120, -9121, -9122, -9123, -9124, -9125, -9126, -9127, -9200, -9201, -9202, -9203, -9501, -9502, -9503, -9504, -9505, -9506, -9507, -9508, -9509, -9510, -9512, -9513, -9514, -9515, -9516, -9518, -9519, -9520, -9521, -9522, -9523, -9524, -9525, -9526, -9527, -9528, -9529, -9530, -9531, -9532, -9533, -9534, -9535, -9536, -9537, -9538, -9539, -9540, -9541, -9542, -9543, -9544, -9545, -9546, -9547, -9548, -9549, -9550, -9551, -9552, -9553, -9554, -9555, -9556, -9557, -9558, -9559, -9560, -9561, -9562, -9563, -9564, -9565, -9566, -9567, -9568, -9569, -9570, -9571, -9572, -9573, -9574, -9575, -9576, -9577, -9578, -9579, -9580, -9581, -9582, -9583, -9584, -9585, -9586, -9587, -9588, -9589, -9590, -9591, -9592, -9593, -9594, -9595, -9596, -9597, -9598, -9599, -9600, -9601, -9602, -9603, -9604, -9605, -9606, -9607, -9608, -9609, -9610, -9611, -9612, -9613, -9614, -9615, -9616, -9617, -9618, -9619, -9620, -9621, -9622, -9623, -9624, -9625, -9626, -9627, -9628, -9629, -9630, -9631, -9632, -9633, -9634, -9635, -9636, -9637, -9638, -9639, -9640, -9641, -9642, -9643, -9644, -9645, -9646, -9647, -9648, -9649, -9650, -9651, -9652, -9653, -9654, -9655, -9656, -9657, -9658, -9659, -9660, -9661, -9662, -9663, -9664, -9665, -9666, -9667, -9668, -9669, -9670, -9671, -9672, -9673, -9674, -9675, -9676, -9677, -9678, -9679, -9680, -9681, -9682, -9683, -9684, -9685, -9686, -9687, -9688, -9689, -9690, -9691, -9692, -9693, -9694, -9695, -9696, -9697, -9698, -9699, -9700, -9701, -9702, -9703, -9704, -9705, -9706, -9707, -9708, -9709, -9710, -9711, -9712, -9713, -9714, -9715, -9716, -9717, -9718, -9719, -9720, -9721, -9722, -9723, -9724, -9725, -9726, -9727, -9728, -9729, -9730, -9731, -9732, -9733, -9734, -9735, -9736, -9737, -9738, -9739, -9740, -9741, -9742, -9743, -9744, -9745, -9746, -9747, -9748, -9749, -9750, -9751, -9752, -9753, -9754, -9755, -9756, -9757, -9758, -9759, -9760, -9761, -9762, -9763, -9764, -9765, -9766, -9767, -9768, -9769, -9770, -9771, -9772, -9773, -9774, -9775, -9776, -9777, -9778, -9779, -9780, -9781, -9782, -9783, -9784, -9785, -9786, -9787, -9788, -9789, -9790, -9791, -9792, -9793, -9794, -9795, -9796, -9797, -9798, -9799, -10500, -10501, -10502, -10503, -10504, -10505, -10506, -10507, -10508, -10509, -10510, -10511, -10512, -10513, -10514, -10515, -10516, -10517, -10650, -11000, -11001, -11002, -11003, -11004, -11005, -11006, -11007, -11008, -11009, -11010, -11011, -11012, -11013, -11014, -11015, -11016, -11017, -11018, -11019, -11020, -11021, -11022, -11023, -11024, -11025, -11026, -11027, -11028, -11029, -11030, -11031, -11032, -11033, -11034, -11035, -11036, -11037, -11038, -11039, -11040, -11041, -11042, -11043, -11044, -11045, -11046, -11047, -11048, -11049, -11050, -11051, -11052, -11053, -11054, -11055, -11056, -11057, -11058, -11059, -11060, -11061, -11062, -20000, -21000, -22998, -30926, -32491, -38104, -38105}; +int g_all_ob_errnos[2357] = {0, -4000, -4001, -4002, -4003, -4004, -4005, -4006, -4007, -4008, -4009, -4010, -4011, -4012, -4013, -4014, -4015, -4016, -4017, -4018, -4019, -4020, -4021, -4022, -4023, -4024, -4025, -4026, -4027, -4028, -4029, -4030, -4031, -4032, -4033, -4034, -4035, -4036, -4037, -4038, -4039, -4041, -4042, -4043, -4044, -4045, -4046, -4047, -4048, -4049, -4050, -4051, -4052, -4053, -4054, -4055, -4057, -4058, -4060, -4061, -4062, -4063, -4064, -4065, -4066, -4067, -4068, -4070, -4071, -4072, -4073, -4074, -4075, -4076, -4077, -4078, -4080, -4081, -4084, -4085, -4090, -4097, -4098, -4099, -4100, -4101, -4102, -4103, -4104, -4105, -4106, -4107, -4108, -4109, -4110, -4111, -4112, -4113, -4114, -4115, -4116, -4117, -4118, -4119, -4120, -4121, -4122, -4123, -4124, -4125, -4126, -4127, -4128, -4133, -4138, -4139, -4142, -4143, -4144, -4146, -4147, -4149, -4150, -4151, -4152, -4153, -4154, -4155, -4156, -4157, -4158, -4159, -4160, -4161, -4162, -4163, -4164, -4165, -4166, -4167, -4168, -4169, -4170, -4171, -4172, -4173, -4174, -4175, -4176, -4177, -4178, -4179, -4180, -4181, -4182, -4183, -4184, -4185, -4186, -4187, -4188, -4189, -4190, -4191, -4192, -4200, -4201, -4204, -4205, -4206, -4207, -4208, -4209, -4210, -4211, -4212, -4213, -4214, -4215, -4216, -4217, -4218, -4219, -4220, -4221, -4222, -4223, -4224, -4225, -4226, -4227, -4228, -4229, -4230, -4231, -4232, -4233, -4234, -4235, -4236, -4237, -4238, -4239, -4240, -4241, -4242, -4243, -4244, -4245, -4246, -4247, -4248, -4249, -4250, -4251, -4252, -4253, -4254, -4255, -4256, -4257, -4258, -4260, -4261, -4262, -4263, -4264, -4265, -4266, -4267, -4268, -4269, -4270, -4271, -4273, -4274, -4275, -4276, -4277, -4278, -4279, -4280, -4281, -4282, -4283, -4284, -4285, -4286, -4287, -4288, -4289, -4290, -4291, -4292, -4293, -4294, -4295, -4296, -4297, -4298, -4299, -4300, -4301, -4302, -4303, -4304, -4305, -4306, -4307, -4308, -4309, -4310, -4311, -4312, -4313, -4314, -4315, -4316, -4317, -4318, -4319, -4320, -4321, -4322, -4323, -4324, -4325, -4326, -4327, -4328, -4329, -4330, -4331, -4332, -4333, -4334, -4335, -4336, -4337, -4338, -4339, -4340, -4341, -4342, -4343, -4344, -4345, -4346, -4347, -4348, -4349, -4350, -4351, -4352, -4353, -4354, -4355, -4356, -4357, -4358, -4359, -4360, -4361, -4362, -4363, -4364, -4365, -4366, -4367, -4368, -4369, -4370, -4371, -4372, -4373, -4374, -4375, -4376, -4377, -4378, -4379, -4380, -4381, -4382, -4383, -4385, -4386, -4387, -4388, -4389, -4390, -4391, -4392, -4393, -4394, -4395, -4396, -4397, -4398, -4399, -4400, -4401, -4402, -4403, -4404, -4505, -4507, -4510, -4512, -4515, -4517, -4518, -4519, -4523, -4524, -4525, -4526, -4527, -4528, -4529, -4530, -4531, -4532, -4533, -4537, -4538, -4539, -4540, -4541, -4542, -4543, -4544, -4545, -4546, -4547, -4548, -4549, -4550, -4551, -4552, -4553, -4554, -4600, -4601, -4602, -4603, -4604, -4605, -4606, -4607, -4608, -4609, -4610, -4611, -4613, -4614, -4615, -4620, -4621, -4622, -4623, -4624, -4625, -4626, -4628, -4629, -4630, -4631, -4632, -4633, -4634, -4636, -4637, -4638, -4639, -4640, -4641, -4642, -4643, -4644, -4645, -4646, -4647, -4648, -4649, -4650, -4651, -4652, -4653, -4654, -4655, -4656, -4657, -4658, -4659, -4660, -4661, -4662, -4663, -4664, -4665, -4666, -4667, -4668, -4669, -4670, -4671, -4672, -4673, -4674, -4675, -4676, -4677, -4678, -4679, -4680, -4681, -4682, -4683, -4684, -4685, -4686, -4687, -4688, -4689, -4690, -4691, -4692, -4693, -4694, -4695, -4696, -4697, -4698, -4699, -4700, -4701, -4702, -4703, -4704, -4705, -4706, -4707, -4708, -4709, -4710, -4711, -4712, -4713, -4714, -4715, -4716, -4717, -4718, -4719, -4720, -4721, -4722, -4723, -4724, -4725, -4726, -4727, -4728, -4729, -4730, -4731, -4732, -4733, -4734, -4735, -4736, -4737, -4738, -4739, -4740, -4741, -4742, -4743, -4744, -4745, -4746, -4747, -4748, -4749, -4750, -4751, -4752, -4753, -4754, -4755, -4756, -4757, -4758, -4759, -4760, -4761, -4762, -4763, -4764, -4765, -4766, -4767, -4768, -4769, -4770, -4771, -4772, -4773, -4774, -4775, -4776, -4777, -4778, -4779, -4780, -4781, -4782, -4783, -4784, -4785, -5000, -5001, -5002, -5003, -5006, -5007, -5008, -5010, -5011, -5012, -5014, -5015, -5016, -5017, -5018, -5019, -5020, -5022, -5023, -5024, -5025, -5026, -5027, -5028, -5029, -5030, -5031, -5032, -5034, -5035, -5036, -5037, -5038, -5039, -5040, -5041, -5042, -5043, -5044, -5046, -5047, -5050, -5051, -5052, -5053, -5054, -5055, -5056, -5057, -5058, -5059, -5061, -5063, -5064, -5065, -5066, -5067, -5068, -5069, -5070, -5071, -5072, -5073, -5074, -5080, -5081, -5083, -5084, -5085, -5086, -5087, -5088, -5089, -5090, -5091, -5092, -5093, -5094, -5095, -5096, -5097, -5098, -5099, -5100, -5101, -5102, -5103, -5104, -5105, -5106, -5107, -5108, -5109, -5110, -5111, -5112, -5113, -5114, -5115, -5116, -5117, -5118, -5119, -5120, -5121, -5122, -5123, -5124, -5125, -5130, -5131, -5133, -5134, -5135, -5136, -5137, -5138, -5139, -5140, -5142, -5143, -5144, -5145, -5146, -5147, -5148, -5149, -5150, -5151, -5153, -5154, -5155, -5156, -5157, -5158, -5159, -5160, -5161, -5162, -5163, -5164, -5165, -5166, -5167, -5168, -5169, -5170, -5171, -5172, -5173, -5174, -5175, -5176, -5177, -5178, -5179, -5180, -5181, -5182, -5183, -5184, -5185, -5187, -5188, -5189, -5190, -5191, -5192, -5193, -5194, -5195, -5196, -5197, -5198, -5199, -5200, -5201, -5202, -5203, -5204, -5205, -5206, -5207, -5208, -5209, -5210, -5211, -5212, -5213, -5214, -5215, -5216, -5217, -5218, -5219, -5220, -5221, -5222, -5223, -5224, -5225, -5226, -5227, -5228, -5229, -5230, -5231, -5233, -5234, -5235, -5236, -5237, -5238, -5239, -5240, -5241, -5242, -5243, -5244, -5245, -5246, -5247, -5248, -5249, -5250, -5251, -5252, -5253, -5254, -5255, -5256, -5257, -5258, -5259, -5260, -5261, -5262, -5263, -5264, -5265, -5266, -5267, -5268, -5269, -5270, -5271, -5272, -5273, -5274, -5275, -5276, -5277, -5278, -5279, -5280, -5281, -5282, -5283, -5284, -5285, -5286, -5287, -5288, -5289, -5290, -5291, -5292, -5293, -5294, -5295, -5296, -5297, -5298, -5299, -5300, -5301, -5302, -5303, -5304, -5305, -5306, -5307, -5308, -5309, -5310, -5311, -5312, -5313, -5314, -5315, -5316, -5317, -5318, -5319, -5320, -5321, -5322, -5323, -5324, -5325, -5326, -5327, -5328, -5329, -5330, -5331, -5332, -5333, -5334, -5335, -5336, -5337, -5338, -5339, -5340, -5341, -5342, -5343, -5344, -5345, -5346, -5347, -5348, -5349, -5350, -5351, -5352, -5353, -5354, -5355, -5356, -5357, -5358, -5359, -5360, -5361, -5362, -5363, -5364, -5365, -5366, -5367, -5368, -5369, -5370, -5371, -5372, -5373, -5374, -5375, -5376, -5377, -5378, -5379, -5380, -5381, -5382, -5383, -5384, -5385, -5386, -5387, -5388, -5389, -5390, -5400, -5401, -5402, -5403, -5404, -5405, -5406, -5407, -5408, -5409, -5410, -5411, -5412, -5413, -5414, -5415, -5416, -5417, -5418, -5419, -5420, -5421, -5422, -5423, -5424, -5425, -5426, -5427, -5428, -5429, -5430, -5431, -5432, -5433, -5434, -5435, -5436, -5437, -5438, -5439, -5440, -5441, -5442, -5443, -5444, -5445, -5446, -5447, -5448, -5449, -5450, -5451, -5452, -5453, -5454, -5455, -5456, -5457, -5458, -5459, -5460, -5461, -5462, -5463, -5464, -5465, -5466, -5467, -5468, -5469, -5470, -5471, -5472, -5473, -5474, -5475, -5476, -5477, -5478, -5479, -5480, -5481, -5482, -5483, -5484, -5485, -5486, -5487, -5488, -5489, -5490, -5491, -5492, -5493, -5494, -5495, -5496, -5497, -5498, -5499, -5500, -5501, -5502, -5503, -5504, -5505, -5506, -5507, -5508, -5509, -5510, -5511, -5512, -5513, -5514, -5515, -5516, -5517, -5518, -5519, -5520, -5521, -5522, -5540, -5541, -5542, -5543, -5544, -5545, -5546, -5547, -5548, -5549, -5550, -5551, -5552, -5553, -5554, -5555, -5556, -5557, -5558, -5559, -5560, -5561, -5562, -5563, -5564, -5565, -5566, -5567, -5568, -5569, -5570, -5571, -5572, -5573, -5574, -5575, -5576, -5577, -5578, -5579, -5580, -5581, -5582, -5583, -5584, -5585, -5586, -5587, -5588, -5589, -5590, -5591, -5592, -5593, -5594, -5595, -5596, -5597, -5598, -5599, -5600, -5601, -5602, -5603, -5604, -5605, -5607, -5608, -5609, -5610, -5611, -5612, -5613, -5614, -5615, -5616, -5617, -5618, -5619, -5620, -5621, -5622, -5623, -5624, -5625, -5626, -5627, -5628, -5629, -5630, -5631, -5632, -5633, -5634, -5635, -5636, -5637, -5638, -5639, -5640, -5641, -5642, -5643, -5644, -5645, -5646, -5647, -5648, -5649, -5650, -5651, -5652, -5653, -5654, -5655, -5656, -5657, -5658, -5659, -5660, -5661, -5662, -5663, -5664, -5665, -5666, -5667, -5668, -5671, -5672, -5673, -5674, -5675, -5676, -5677, -5678, -5679, -5680, -5681, -5682, -5683, -5684, -5685, -5686, -5687, -5688, -5689, -5690, -5691, -5692, -5693, -5694, -5695, -5696, -5697, -5698, -5699, -5700, -5701, -5702, -5703, -5704, -5705, -5706, -5707, -5708, -5709, -5710, -5711, -5712, -5713, -5714, -5715, -5716, -5717, -5718, -5719, -5720, -5721, -5722, -5723, -5724, -5725, -5726, -5727, -5728, -5729, -5730, -5731, -5732, -5733, -5734, -5735, -5736, -5737, -5738, -5739, -5740, -5741, -5742, -5743, -5744, -5745, -5746, -5747, -5748, -5749, -5750, -5751, -5752, -5753, -5754, -5755, -5756, -5757, -5758, -5759, -5760, -5761, -5762, -5763, -5764, -5765, -5766, -5768, -5769, -5770, -5771, -5772, -5773, -5774, -5777, -5778, -5779, -5780, -5781, -5785, -5786, -5787, -5788, -5789, -5790, -5791, -5792, -5793, -5794, -5795, -5796, -5797, -5798, -5799, -5800, -5801, -5802, -5803, -5804, -5805, -5806, -5807, -5808, -5809, -5810, -5811, -5812, -5813, -5814, -5815, -5816, -5817, -5818, -5819, -5820, -5821, -5822, -5823, -5824, -5825, -5826, -5827, -5828, -5829, -5830, -5831, -5832, -5833, -5834, -5835, -5836, -5837, -5838, -5839, -5840, -5841, -5842, -5843, -5844, -5845, -5846, -5847, -5848, -5849, -5850, -5851, -5852, -5853, -5854, -5855, -5856, -5857, -5858, -5859, -5860, -5861, -5862, -5863, -5864, -5865, -5866, -5867, -5868, -5869, -5870, -5871, -5872, -5873, -5874, -5875, -5876, -5877, -5878, -5879, -5880, -5881, -5882, -5883, -5884, -5885, -5886, -5887, -5888, -5889, -5890, -5891, -5892, -5893, -5894, -5895, -5896, -5897, -5898, -5899, -5900, -5901, -5902, -5903, -5904, -5905, -5906, -5907, -5908, -5909, -5910, -5911, -5912, -5913, -5914, -5915, -5916, -5917, -5918, -5919, -5920, -5921, -5922, -5923, -5924, -5925, -5926, -5927, -5928, -5929, -5930, -5931, -5932, -5933, -5934, -5935, -5936, -5937, -5938, -5939, -5940, -5941, -5942, -5943, -5944, -5945, -5946, -5947, -5948, -5949, -5950, -5951, -5952, -5953, -5954, -5955, -5956, -5957, -5958, -5959, -5960, -5961, -5962, -5963, -5964, -5965, -5966, -5967, -5968, -5969, -5970, -5971, -5972, -5973, -5974, -5975, -5976, -5977, -5978, -5979, -5980, -5981, -5982, -5983, -5984, -5985, -5986, -5987, -5988, -5989, -5990, -5991, -5992, -5993, -5994, -5995, -5996, -5997, -5998, -5999, -6000, -6001, -6002, -6003, -6004, -6005, -6006, -6201, -6202, -6203, -6204, -6205, -6206, -6207, -6208, -6209, -6210, -6211, -6212, -6213, -6214, -6215, -6219, -6220, -6221, -6222, -6223, -6224, -6225, -6226, -6227, -6228, -6229, -6230, -6231, -6232, -6233, -6234, -6235, -6236, -6237, -6238, -6239, -6240, -6241, -6242, -6243, -6244, -6245, -6246, -6247, -6248, -6249, -6250, -6251, -6252, -6253, -6254, -6255, -6256, -6257, -6258, -6259, -6260, -6261, -6262, -6263, -6264, -6265, -6266, -6267, -6268, -6269, -6270, -6271, -6272, -6273, -6274, -6275, -6276, -6277, -6278, -6279, -6280, -6281, -6282, -6283, -6284, -6285, -6301, -6302, -6303, -6304, -6305, -6306, -6307, -6308, -6309, -6310, -6311, -6312, -6313, -6314, -6315, -6316, -6317, -6318, -6319, -6320, -6321, -6322, -6323, -6324, -6325, -6326, -6327, -6328, -6329, -6330, -6331, -6332, -7000, -7001, -7002, -7003, -7004, -7005, -7006, -7007, -7010, -7011, -7012, -7013, -7014, -7015, -7021, -7022, -7024, -7025, -7026, -7027, -7029, -7030, -7031, -7032, -7033, -7034, -7035, -7036, -7037, -7038, -7039, -7040, -7041, -7100, -7101, -7102, -7103, -7104, -7105, -7106, -7107, -7108, -7109, -7110, -7111, -7112, -7113, -7114, -7115, -7116, -7117, -7118, -7119, -7120, -7121, -7122, -7123, -7124, -7125, -7201, -7202, -7203, -7204, -7205, -7206, -7207, -7208, -7209, -7210, -7211, -7212, -7213, -7214, -7215, -7216, -7217, -7218, -7219, -7220, -7221, -7222, -7223, -7224, -7225, -7226, -7227, -7228, -7229, -7230, -7231, -7232, -7233, -7234, -7235, -7236, -7237, -7238, -7239, -7240, -7241, -7242, -7243, -7244, -7246, -7247, -7248, -7249, -7250, -7251, -7252, -7253, -7254, -7255, -7256, -7257, -7258, -7259, -7260, -7261, -7262, -7263, -7264, -7265, -7266, -7267, -7268, -7269, -7270, -7271, -7272, -7273, -7274, -7275, -7276, -7277, -7278, -7279, -7280, -7281, -7282, -7283, -7284, -7285, -7286, -7287, -7288, -7289, -7290, -7291, -7292, -7293, -7294, -7295, -7296, -7297, -7298, -7299, -7300, -7301, -7302, -7402, -7403, -7404, -7405, -7406, -7407, -7408, -7409, -7410, -7411, -7412, -7413, -7414, -7415, -7416, -7417, -7418, -7419, -7420, -7421, -7422, -7423, -7424, -7425, -7426, -7427, -7428, -7429, -7430, -7431, -7432, -7433, -7434, -7435, -7600, -7601, -7602, -7603, -7604, -8001, -8002, -8003, -8004, -8005, -9001, -9002, -9003, -9004, -9005, -9006, -9007, -9008, -9009, -9010, -9011, -9012, -9013, -9014, -9015, -9016, -9017, -9018, -9019, -9020, -9022, -9023, -9024, -9025, -9026, -9027, -9028, -9029, -9030, -9031, -9032, -9033, -9034, -9035, -9036, -9037, -9038, -9039, -9040, -9041, -9042, -9043, -9044, -9045, -9046, -9047, -9048, -9049, -9050, -9051, -9052, -9053, -9054, -9057, -9058, -9059, -9060, -9061, -9062, -9063, -9064, -9065, -9066, -9069, -9070, -9071, -9072, -9073, -9074, -9075, -9076, -9077, -9078, -9079, -9080, -9081, -9082, -9083, -9084, -9085, -9086, -9087, -9088, -9089, -9090, -9091, -9092, -9093, -9094, -9095, -9096, -9097, -9098, -9099, -9100, -9101, -9102, -9103, -9104, -9105, -9106, -9107, -9108, -9109, -9110, -9111, -9112, -9113, -9114, -9115, -9116, -9117, -9118, -9119, -9120, -9121, -9122, -9123, -9124, -9125, -9126, -9127, -9200, -9201, -9202, -9203, -9501, -9502, -9503, -9504, -9505, -9506, -9507, -9508, -9509, -9510, -9512, -9513, -9514, -9515, -9516, -9518, -9519, -9520, -9521, -9522, -9523, -9524, -9525, -9526, -9527, -9528, -9529, -9530, -9531, -9532, -9533, -9534, -9535, -9536, -9537, -9538, -9539, -9540, -9541, -9542, -9543, -9544, -9545, -9546, -9547, -9548, -9549, -9550, -9551, -9552, -9553, -9554, -9555, -9556, -9557, -9558, -9559, -9560, -9561, -9562, -9563, -9564, -9565, -9566, -9567, -9568, -9569, -9570, -9571, -9572, -9573, -9574, -9575, -9576, -9577, -9578, -9579, -9580, -9581, -9582, -9583, -9584, -9585, -9586, -9587, -9588, -9589, -9590, -9591, -9592, -9593, -9594, -9595, -9596, -9597, -9598, -9599, -9600, -9601, -9602, -9603, -9604, -9605, -9606, -9607, -9608, -9609, -9610, -9611, -9612, -9613, -9614, -9615, -9616, -9617, -9618, -9619, -9620, -9621, -9622, -9623, -9624, -9625, -9626, -9627, -9628, -9629, -9630, -9631, -9632, -9633, -9634, -9635, -9636, -9637, -9638, -9639, -9640, -9641, -9642, -9643, -9644, -9645, -9646, -9647, -9648, -9649, -9650, -9651, -9652, -9653, -9654, -9655, -9656, -9657, -9658, -9659, -9660, -9661, -9662, -9663, -9664, -9665, -9666, -9667, -9668, -9669, -9670, -9671, -9672, -9673, -9674, -9675, -9676, -9677, -9678, -9679, -9680, -9681, -9682, -9683, -9684, -9685, -9686, -9687, -9688, -9689, -9690, -9691, -9692, -9693, -9694, -9695, -9696, -9697, -9698, -9699, -9700, -9701, -9702, -9703, -9704, -9705, -9706, -9707, -9708, -9709, -9710, -9711, -9712, -9713, -9714, -9715, -9716, -9717, -9718, -9719, -9720, -9721, -9722, -9723, -9724, -9725, -9726, -9727, -9728, -9729, -9730, -9731, -9732, -9733, -9734, -9735, -9736, -9737, -9738, -9739, -9740, -9741, -9742, -9743, -9744, -9745, -9746, -9747, -9748, -9749, -9750, -9751, -9752, -9753, -9754, -9755, -9756, -9757, -9758, -9759, -9760, -9761, -9762, -9763, -9764, -9765, -9766, -9767, -9768, -9769, -9770, -9771, -9772, -9773, -9774, -9775, -9776, -9777, -9778, -9779, -9780, -9781, -9782, -9783, -9784, -9785, -9786, -9787, -9788, -9789, -9790, -9791, -9792, -9793, -9794, -9795, -9796, -9797, -9798, -9799, -10500, -10501, -10502, -10503, -10504, -10505, -10506, -10507, -10508, -10509, -10510, -10511, -10512, -10513, -10514, -10515, -10516, -10517, -10650, -11000, -11001, -11002, -11003, -11004, -11005, -11006, -11007, -11008, -11009, -11010, -11011, -11012, -11013, -11014, -11015, -11016, -11017, -11018, -11019, -11020, -11021, -11022, -11023, -11024, -11025, -11026, -11027, -11028, -11029, -11030, -11031, -11032, -11033, -11034, -11035, -11036, -11037, -11038, -11039, -11040, -11041, -11042, -11043, -11044, -11045, -11046, -11047, -11048, -11049, -11050, -11051, -11052, -11053, -11054, -11055, -11056, -11057, -11058, -11059, -11060, -11061, -11062, -20000, -21000, -22998, -30926, -32491, -38104, -38105}; const char *ob_error_name(const int err) { const char *ret = "Unknown error"; diff --git a/src/share/ob_errno.def b/src/share/ob_errno.def index 06336b4a92..b0b69149c3 100755 --- a/src/share/ob_errno.def +++ b/src/share/ob_errno.def @@ -1957,6 +1957,7 @@ DEFINE_ERROR(OB_SEQUENCE_NOT_MATCH, -7121, -1, "HY000", "compare sequence not ma DEFINE_ERROR(OB_SEQUENCE_TOO_SMALL, -7122, -1, "HY000", "compare sequence too small"); DEFINE_ERROR(OB_TRANSFER_CANNOT_START, -7123, -1, "HY000", "transfer cannot start"); DEFINE_ERROR_EXT_DEP(OB_PARTITION_ALREADY_BALANCED, -7124, -1, "HY000", "partitions are already balanced", "partitions are already balanced, %s"); +DEFINE_ERROR(NEW_MV_MAJOR_VERSION_NOT_MATCH, -7125, -1, "HY000", "new mv major version is not match"); // 余留位置 //////////////////////////////////////////////////////////////// diff --git a/src/share/ob_errno.h b/src/share/ob_errno.h index 87bd887742..4a9d74b6c2 100644 --- a/src/share/ob_errno.h +++ b/src/share/ob_errno.h @@ -1466,6 +1466,7 @@ constexpr int OB_TABLET_GC_LOCK_CONFLICT = -7120; constexpr int OB_SEQUENCE_NOT_MATCH = -7121; constexpr int OB_SEQUENCE_TOO_SMALL = -7122; constexpr int OB_TRANSFER_CANNOT_START = -7123; +constexpr int NEW_MV_MAJOR_VERSION_NOT_MATCH = -7125; constexpr int OB_ERR_DIMENSION_NUMBER_IS_OUT_OF_RANGE = -7290; constexpr int OB_ERR_INVALID_SRID_IN_SDO_GEOMETRY = -7292; constexpr int OB_ERR_INVALID_GTYPE_FOR_POINT_OBJECT = -7293; @@ -3643,6 +3644,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_SEQUENCE_TOO_SMALL__USER_ERROR_MSG "compare sequence too small" #define OB_TRANSFER_CANNOT_START__USER_ERROR_MSG "transfer cannot start" #define OB_PARTITION_ALREADY_BALANCED__USER_ERROR_MSG "partitions are already balanced, %s" +#define NEW_MV_MAJOR_VERSION_NOT_MATCH__USER_ERROR_MSG "new mv major version is not match" #define OB_ERR_GIS_DIFFERENT_SRIDS__USER_ERROR_MSG "Binary geometry function %s given two geometries of different srids: %u and %u, which should have been identical." #define OB_ERR_GIS_UNSUPPORTED_ARGUMENT__USER_ERROR_MSG "Calling geometry function %s with unsupported types of arguments." #define OB_ERR_GIS_UNKNOWN_ERROR__USER_ERROR_MSG "Unknown GIS error occurred in function %s." @@ -7702,6 +7704,8 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_TRANSFER_CANNOT_START__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -7123, transfer cannot start" #define OB_PARTITION_ALREADY_BALANCED__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -7124, partitions are already balanced, %s" #define OB_PARTITION_ALREADY_BALANCED__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -7124, partitions are already balanced, %s" +#define NEW_MV_MAJOR_VERSION_NOT_MATCH__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -7125, new mv major version is not match" +#define NEW_MV_MAJOR_VERSION_NOT_MATCH__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -7125, new mv major version is not match" #define OB_ERR_GIS_DIFFERENT_SRIDS__ORA_USER_ERROR_MSG "ORA-00600: Binary geometry function %s given two geometries of different srids: %u and %u, which should have been identical." #define OB_ERR_GIS_DIFFERENT_SRIDS__OBE_USER_ERROR_MSG "OBE-00600: Binary geometry function %s given two geometries of different srids: %u and %u, which should have been identical." #define OB_ERR_GIS_UNSUPPORTED_ARGUMENT__ORA_USER_ERROR_MSG "ORA-00600: Calling geometry function %s with unsupported types of arguments." @@ -9021,7 +9025,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_INVALID_DATE_MSG_FMT_V2__ORA_USER_ERROR_MSG "ORA-01861: Incorrect datetime value for column '%.*s' at row %ld" #define OB_ERR_INVALID_DATE_MSG_FMT_V2__OBE_USER_ERROR_MSG "OBE-01861: Incorrect datetime value for column '%.*s' at row %ld" -extern int g_all_ob_errnos[2356]; +extern int g_all_ob_errnos[2357]; const char *ob_error_name(const int oberr); const char* ob_error_cause(const int oberr); diff --git a/src/share/ob_global_stat_proxy.cpp b/src/share/ob_global_stat_proxy.cpp index 1fabc30a6c..a0bf861689 100644 --- a/src/share/ob_global_stat_proxy.cpp +++ b/src/share/ob_global_stat_proxy.cpp @@ -301,6 +301,38 @@ int ObGlobalStatProxy::get_finish_data_version(uint64_t &finish_data_version, return ret; } +int ObGlobalStatProxy::update_major_refresh_mv_merge_scn(const share::SCN &scn, bool is_incremental) +{ + int ret = OB_SUCCESS; + ObGlobalStatItem::ItemList list; + ObGlobalStatItem scn_item(list, "major_refresh_mv_merge_scn", + scn.get_val_for_inner_table_field()); + if (!is_valid() || !scn.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), "valid", is_valid(), K(scn)); + } else if (OB_FAIL(update(list, is_incremental))) { + LOG_WARN("update failed", KR(ret), K(list)); + } + return ret; +} + +int ObGlobalStatProxy::get_major_refresh_mv_merge_scn(const bool for_update, share::SCN &scn) +{ + int ret = OB_SUCCESS; + scn.reset(); + ObGlobalStatItem::ItemList list; + ObGlobalStatItem scn_item(list, "major_refresh_mv_merge_scn", OB_INVALID_SCN_VAL); + if (!is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), "self valid", is_valid()); + } else if (OB_FAIL(get(list, for_update))) { + LOG_WARN("get failed", KR(ret)); + } else { + scn.convert_for_inner_table_field(static_cast(scn_item.value_)); + } + return ret; +} + int ObGlobalStatProxy::get_target_data_version_ora_rowscn( const uint64_t tenant_id, share::SCN &target_data_version_ora_rowscn) diff --git a/src/share/ob_global_stat_proxy.h b/src/share/ob_global_stat_proxy.h index d6b26fa744..6052fe5d8f 100644 --- a/src/share/ob_global_stat_proxy.h +++ b/src/share/ob_global_stat_proxy.h @@ -108,6 +108,10 @@ public: const uint64_t tenant_id, int64_t &ddl_epoch); int get_ddl_epoch(int64_t &ddl_epoch); + // for major refresh mv + int update_major_refresh_mv_merge_scn(const share::SCN &scn, bool is_incremental = true); + int get_major_refresh_mv_merge_scn(const bool for_update, share::SCN &scn); + private: static int inner_get_snapshot_gc_scn_(common::ObISQLClient &sql_client, const uint64_t tenant_id, diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index 594068d129..c94d7bed58 100644 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -9470,7 +9470,8 @@ bool ObCreateLSArg::is_valid() const && create_scn_.is_valid() && lib::Worker::CompatMode::INVALID != compat_mode_ && (!is_create_ls_with_palf() - || palf_base_info_.is_valid()); + || palf_base_info_.is_valid()) + && major_mv_merge_info_.is_valid(); } void ObCreateLSArg::reset() { @@ -9512,16 +9513,18 @@ int ObCreateLSArg::init(const int64_t tenant_id, const SCN &create_scn, const lib::Worker::CompatMode &mode, const bool create_with_palf, - const palf::PalfBaseInfo &palf_base_info) + const palf::PalfBaseInfo &palf_base_info, + const storage::ObMajorMVMergeInfo &major_mv_merge_info) { int ret = OB_SUCCESS; if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id ||!id.is_valid() || !ObReplicaTypeCheck::is_replica_type_valid(replica_type) || !replica_property.is_valid() - || !tenant_info.is_valid())) { + || !tenant_info.is_valid() + || !major_mv_merge_info.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(id), K(replica_type), K(tenant_info)); + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(id), K(replica_type), K(tenant_info), K(major_mv_merge_info)); } else { (void)tenant_info_.assign(tenant_info); tenant_id_ = tenant_id; @@ -9536,6 +9539,7 @@ int ObCreateLSArg::init(const int64_t tenant_id, create_ls_type_ = EMPTY_LS; } palf_base_info_ = palf_base_info; + major_mv_merge_info_ = major_mv_merge_info; } return ret; } @@ -9544,7 +9548,7 @@ DEF_TO_STRING(ObCreateLSArg) { int64_t pos = 0; J_KV(K_(tenant_id), K_(id), K_(replica_type), K_(replica_property), K_(tenant_info), - K_(create_scn), K_(compat_mode), K_(palf_base_info), "create with palf", is_create_ls_with_palf()); + K_(create_scn), K_(compat_mode), K_(palf_base_info), "create with palf", is_create_ls_with_palf(), K_(major_mv_merge_info)); return pos; } @@ -12997,6 +13001,84 @@ int ObCheckServerMachineStatusResult::assign(const ObCheckServerMachineStatusRes } return ret; } + +OB_SERIALIZE_MEMBER(ObCollectMvMergeInfoArg, ls_id_, + tenant_id_, check_leader_, need_update_); +int ObCollectMvMergeInfoArg::assign(const ObCollectMvMergeInfoArg &other) +{ + int ret = OB_SUCCESS; + if (this != &other) { + ls_id_ = other.ls_id_; + tenant_id_ = other.tenant_id_; + check_leader_ = other.check_leader_; + need_update_ = other.need_update_; + } + return ret; +} +int ObCollectMvMergeInfoArg::init(const share::ObLSID &ls_id, + const uint64_t tenant_id, + const bool check_leader, + const bool need_update) +{ + int ret = OB_SUCCESS; + + if (!ls_id.is_valid() || OB_INVALID_TENANT_ID == tenant_id) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", KR(ret), K(ls_id), K(tenant_id)); + } else { + ls_id_ = ls_id; + tenant_id_ = tenant_id; + check_leader_ = check_leader; + need_update_ = need_update; + } + return ret; +} + +OB_SERIALIZE_MEMBER(ObCollectMvMergeInfoResult, mv_merge_info_, ret_); +int ObCollectMvMergeInfoResult::assign(const ObCollectMvMergeInfoResult &other) +{ + int ret = OB_SUCCESS; + if (this != &other) { + mv_merge_info_ = other.mv_merge_info_; + } + return ret; +} +int ObCollectMvMergeInfoResult::init(const ObMajorMVMergeInfo &mv_merge_info, const int err_ret) +{ + int ret = OB_SUCCESS; + if (!mv_merge_info.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", KR(ret), K(mv_merge_info)); + } else { + mv_merge_info_ = mv_merge_info; + ret_ = err_ret; + } + return ret; +} + +OB_SERIALIZE_MEMBER(ObFetchStableMemberListArg, tenant_id_, ls_id_); +int ObFetchStableMemberListArg::assign(const ObFetchStableMemberListArg &other) +{ + int ret = OB_SUCCESS; + if (this != &other) { + tenant_id_ = other.tenant_id_; + ls_id_ = other.ls_id_; + } + return ret; +} +int ObFetchStableMemberListArg::init(const share::ObLSID &ls_id, const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + if (!ls_id.is_valid() || OB_INVALID_TENANT_ID == tenant_id) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", KR(ret), K(ls_id), K(tenant_id)); + } else { + ls_id_ = ls_id; + tenant_id_ = tenant_id; + } + return ret; +} + OB_SERIALIZE_MEMBER(ObRefreshServiceNameArg, tenant_id_, epoch_, from_server_, target_service_name_id_, service_name_list_, service_op_, update_tenant_info_arg_); int ObRefreshServiceNameArg::init( @@ -13127,6 +13209,30 @@ int ObNotifySharedStorageInfoArg::init(const ObIArray &shared return ret; } +OB_SERIALIZE_MEMBER(ObFetchStableMemberListInfo, member_list_, config_version_); + +int ObFetchStableMemberListInfo::init(const common::ObMemberList &member_list, const palf::LogConfigVersion &config_version) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!member_list.is_valid() || !config_version.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", KR(ret), K(member_list), K(config_version)); + } else if (OB_FAIL(member_list_.deep_copy(member_list))) { + LOG_WARN("fail to assign memberlist", KR(ret), K(member_list)); + } else if (OB_FALSE_IT(config_version_ = config_version)) { + } + return ret; +} +int ObFetchStableMemberListInfo::assign(const ObFetchStableMemberListInfo &other) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(member_list_.deep_copy(other.member_list_))) { + LOG_WARN("fail to deep copy memberlist", KR(ret), K(other)); + } else if (OB_FALSE_IT(config_version_ = other.config_version_)) { + } + return ret; +} + int ObNotifySharedStorageInfoArg::assign(const ObNotifySharedStorageInfoArg &other) { int ret = OB_SUCCESS; diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index dbe4b14f8c..bc65170fd5 100644 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -3769,7 +3769,8 @@ public: const share::SCN &create_scn, const lib::Worker::CompatMode &mode, const bool create_with_palf, - const palf::PalfBaseInfo &palf_base_info); + const palf::PalfBaseInfo &palf_base_info, + const storage::ObMajorMVMergeInfo &major_mv_merge_info); int64_t get_tenant_id() const { return tenant_id_; @@ -3808,6 +3809,10 @@ public: { return CREATE_WITH_PALF != create_ls_type_; } + const storage::ObMajorMVMergeInfo& get_major_mv_merge_info() const + { + return major_mv_merge_info_; + } DECLARE_TO_STRING; private: @@ -13206,6 +13211,92 @@ private: share::ObServerHealthStatus server_health_status_; }; +struct ObCollectMvMergeInfoArg final +{ + OB_UNIS_VERSION(1); +public: + ObCollectMvMergeInfoArg() : ls_id_(), + tenant_id_(OB_INVALID_TENANT_ID), + check_leader_(false), + need_update_(false) + {} + ~ObCollectMvMergeInfoArg() {} + bool is_valid() const { return ls_id_.is_valid() && tenant_id_ != OB_INVALID_TENANT_ID; } + int assign(const ObCollectMvMergeInfoArg &other); + int init(const share::ObLSID &ls_id, + const uint64_t tenant_id, + const bool check_leader = false, + const bool need_update = false); + const share::ObLSID &get_ls_id() const { return ls_id_; } + uint64_t get_tenant_id() const { return tenant_id_; } + bool need_check_leader() const { return check_leader_; } + bool need_update() const { return need_update_; } + TO_STRING_KV(K_(ls_id), K_(tenant_id), + K_(check_leader), K_(need_update)); +private: + share::ObLSID ls_id_; + uint64_t tenant_id_; + bool check_leader_; + bool need_update_; +}; + +struct ObCollectMvMergeInfoResult final +{ + OB_UNIS_VERSION(1); +public: + ObCollectMvMergeInfoResult() : mv_merge_info_(), + ret_(OB_SUCCESS) {} + ~ObCollectMvMergeInfoResult() {} + bool is_valid() const { return mv_merge_info_.is_valid(); } + int assign(const ObCollectMvMergeInfoResult &other); + int init(const ObMajorMVMergeInfo &mv_merge_info, const int err_ret); + int64_t get_ret() const { return ret_; } + const share::SCN &get_publish_mv_merge_scn() const { return mv_merge_info_.major_mv_merge_scn_publish_; } + const share::SCN get_mv_merge_scn() const { return mv_merge_info_.major_mv_merge_scn_; } + const storage::ObMajorMVMergeInfo &get_mv_merge_info() const { return mv_merge_info_; } + TO_STRING_KV(K_(mv_merge_info), K_(ret)); +private: + storage::ObMajorMVMergeInfo mv_merge_info_; + int ret_; +}; + +struct ObFetchStableMemberListArg final +{ + OB_UNIS_VERSION(1); +public: + ObFetchStableMemberListArg(): ls_id_(), + tenant_id_(OB_INVALID_TENANT_ID) {} + ~ObFetchStableMemberListArg() {} + bool is_valid() const { return ls_id_.is_valid() && tenant_id_ != OB_INVALID_TENANT_ID; } + int assign(const ObFetchStableMemberListArg &other); + void reset() { ls_id_.reset(); tenant_id_ = OB_INVALID_TENANT_ID; } + int init(const share::ObLSID &ls_id, const uint64_t tenant_id); + const share::ObLSID &get_ls_id() const { return ls_id_; } + uint64_t get_tenant_id() const { return tenant_id_; } + TO_STRING_KV(K_(ls_id), K_(tenant_id)); +private: + share::ObLSID ls_id_; + uint64_t tenant_id_; +}; + +struct ObFetchStableMemberListInfo final +{ + OB_UNIS_VERSION(1); +public: + ObFetchStableMemberListInfo() : member_list_(), config_version_() {} + ~ObFetchStableMemberListInfo() {} + bool is_valid() const { return member_list_.is_valid() && config_version_.is_valid(); } + int assign(const ObFetchStableMemberListInfo &other); + void reset() { member_list_.reset(); } + int init(const common::ObMemberList &member_list, const palf::LogConfigVersion &config_version); + const common::ObMemberList &get_member_list() const { return member_list_; } + const palf::LogConfigVersion &get_config_version() const { return config_version_; } + TO_STRING_KV(K_(member_list)); +private: + common::ObMemberList member_list_; + palf::LogConfigVersion config_version_; +}; + struct ObRefreshServiceNameArg { OB_UNIS_VERSION(1); diff --git a/src/share/ob_snapshot_table_proxy.cpp b/src/share/ob_snapshot_table_proxy.cpp index 2dc2334bb7..1e8df47a5d 100644 --- a/src/share/ob_snapshot_table_proxy.cpp +++ b/src/share/ob_snapshot_table_proxy.cpp @@ -433,6 +433,57 @@ int ObSnapshotTableProxy::get_all_snapshots( return ret; } +int ObSnapshotTableProxy::get_all_snapshots( + ObISQLClient &proxy, + const uint64_t tenant_id, + ObSnapShotType snapshot_type, + ObIArray &snapshots) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + if (!is_valid_tenant_id(tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + ObMySQLResult *result = NULL; + + if (OB_FAIL(sql.assign_fmt("SELECT * FROM %s WHERE tenant_id = '%lu' AND snapshot_type = %d ORDER BY tablet_id, snapshot_scn", + OB_ALL_ACQUIRED_SNAPSHOT_TNAME, + schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), + snapshot_type))) { + LOG_WARN("fail to assign sql", KR(ret), K(tenant_id)); + } else if (OB_FAIL(proxy.read(res, tenant_id, sql.ptr()))) { + LOG_WARN("fail to execute sql", KR(ret), K(sql), K(tenant_id)); + } else if (NULL == (result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get result", KR(ret), K(sql), K(tenant_id)); + } else { + ObSnapshotInfo snapshot; + while (OB_SUCC(ret)) { + snapshot.reset(); + + if (OB_FAIL(result->next())) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next result", KR(ret), K(sql), K(tenant_id)); + } else { + ret = OB_SUCCESS; + break; + } + } else if (OB_FAIL(extract_snapshot(*result, snapshot, tenant_id))) { + LOG_WARN("fail to extract snapshot", KR(ret)); + } else if (OB_FAIL(snapshots.push_back(snapshot))) { + LOG_WARN("fail to push back snapshot info", KR(ret), K(tenant_id)); + } + } + FLOG_INFO("get all snapshots of type", K(ret), K(snapshot_type), K(snapshots.count()), K(snapshots)); + } + } + } + + return ret; +} + int ObSnapshotTableProxy::check_snapshot_valid( const SCN &snapshot_gc_scn, const ObSnapshotInfo &info, @@ -807,5 +858,29 @@ int ObSnapshotTableProxy::get_snapshot_count( return ret; } +int ObSnapshotTableProxy::push_snapshot_for_major_refresh_mv(common::ObISQLClient &proxy, + const uint64_t tenant_id, + const share::SCN &new_snapshot_scn) +{ + int ret = OB_SUCCESS; + int64_t affected_rows = 0; + ObSqlString sql; + uint64_t snapshot_scn_val = new_snapshot_scn.get_val_for_inner_table_field(); + + if (OB_FAIL(sql.assign_fmt( + "UPDATE %s SET snapshot_scn = %ld WHERE snapshot_type = %d AND snapshot_scn < %ld", + OB_ALL_ACQUIRED_SNAPSHOT_TNAME, snapshot_scn_val, SNAPSHOT_FOR_MAJOR_REFRESH_MV, + snapshot_scn_val))) { + LOG_WARN("fail to assign sql", KR(ret), K(tenant_id), K(snapshot_scn_val)); + } else if (OB_FAIL(proxy.write(tenant_id, sql.ptr(), affected_rows))) { + LOG_WARN("fail to write", KR(ret), K(sql)); + } else if (affected_rows < 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected affected rows", KR(ret), K(affected_rows)); + } + + return ret; +} + } // end namespace share } // end namespace oceanbase diff --git a/src/share/ob_snapshot_table_proxy.h b/src/share/ob_snapshot_table_proxy.h index da8fee4ab7..6d828920d7 100644 --- a/src/share/ob_snapshot_table_proxy.h +++ b/src/share/ob_snapshot_table_proxy.h @@ -116,6 +116,10 @@ public: int get_all_snapshots(common::ObISQLClient &proxy, const uint64_t tenant_id, common::ObIArray &snapshots); + int get_all_snapshots(common::ObISQLClient &proxy, + const uint64_t tenant_id, + ObSnapShotType snapshot_type, + common::ObIArray &snapshots); int get_snapshot(common::ObISQLClient &proxy, const uint64_t tenant_id, ObSnapShotType snapshot_type, @@ -147,6 +151,7 @@ public: const uint64_t tenant_id, ObSnapShotType snapshot_type, int64_t &count); + int push_snapshot_for_major_refresh_mv(common::ObISQLClient &proxy, const uint64_t tenant_id, const share::SCN &new_snapshot_scn); private: int gen_event_ts(int64_t &event_ts); int check_snapshot_valid(const SCN &snapshot_gc_scn, diff --git a/src/share/ob_srv_rpc_proxy.h b/src/share/ob_srv_rpc_proxy.h index acfdc48deb..48970cb6cd 100644 --- a/src/share/ob_srv_rpc_proxy.h +++ b/src/share/ob_srv_rpc_proxy.h @@ -295,6 +295,8 @@ public: RPC_S(PR5 phy_res_calculate_by_unit, OB_CAL_UNIT_PHY_RESOURCE, (obrpc::Int64), share::ObMinPhyResourceResult); RPC_S(PR5 rpc_reverse_keepalive, OB_RPC_REVERSE_KEEPALIVE, (obrpc::ObRpcReverseKeepaliveArg), obrpc::ObRpcReverseKeepaliveResp); RPC_AP(PR5 kill_query_client_session, OB_KILL_QUERY_CLIENT_SESSION, (ObKillQueryClientSessionArg), obrpc::Int64); + RPC_AP(PR5 collect_mv_merge_info, OB_COLLECT_MV_MERGE_INFO, (obrpc::ObCollectMvMergeInfoArg), obrpc::ObCollectMvMergeInfoResult); + RPC_S(PR5 fetch_stable_member_list, OB_FETCH_STABLE_MEMBER_LIST, (obrpc::ObFetchStableMemberListArg), obrpc::ObFetchStableMemberListInfo); RPC_S(PR5 notify_shared_storage_info, OB_NOTIFY_SHARED_STORAGE_INFO, (obrpc::ObNotifySharedStorageInfoArg), obrpc::ObNotifySharedStorageInfoResult); }; // end of class ObSrvRpcProxy diff --git a/src/share/scheduler/ob_diagnose_config.h b/src/share/scheduler/ob_diagnose_config.h index df9d440777..378f9ed9f0 100644 --- a/src/share/scheduler/ob_diagnose_config.h +++ b/src/share/scheduler/ob_diagnose_config.h @@ -56,6 +56,8 @@ SUSPECT_INFO_TYPE_DEF(SUSPECT_TABLET_CANT_MERGE, ObDiagnoseInfoPrio::DIAGNOSE_PR SUSPECT_INFO_TYPE_DEF(SUSPECT_UPDATE_TALBET_STATE_FAILED, ObDiagnoseInfoPrio::DIAGNOSE_PRIORITY_HIGH, false, "update tablet state failed", 3, {"compaction_scn", "is_verified", "is_merged"}) #endif +SUSPECT_INFO_TYPE_DEF(SUSPECT_MV_IN_CREATION, ObDiagnoseInfoPrio::DIAGNOSE_PRIORITY_LOW, false, + "materialized view creation has not finished", 2, {"schedule_scn", "is_row_store"}) SUSPECT_INFO_TYPE_DEF(SUSPECT_INFO_TYPE_MAX, ObDiagnoseInfoPrio::DIAGNOSE_PRIORITY_LOW, false, "", 0, {}) #endif diff --git a/src/share/schema/ob_mview_info.cpp b/src/share/schema/ob_mview_info.cpp index 0e3a0df269..b18fdc25da 100644 --- a/src/share/schema/ob_mview_info.cpp +++ b/src/share/schema/ob_mview_info.cpp @@ -459,6 +459,104 @@ int ObMViewInfo::batch_fetch_mview_ids(ObISQLClient &sql_client, uint64_t tenant return ret; } +int ObMViewInfo::update_major_refresh_mview_scn(ObISQLClient &sql_client, + const uint64_t tenant_id, const share::SCN &scn) +{ + int ret = OB_SUCCESS; + + if (!scn.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid scn", KR(ret), K(scn)); + } else { + const uint64_t scn_val = scn.get_val_for_inner_table_field(); + const int64_t last_refresh_type = (int64_t)ObMVRefreshType::FAST; + int64_t affected_rows = 0; + ObSqlString sql; + if (OB_FAIL(sql.assign_fmt("UPDATE %s SET last_refresh_scn = %lu, \ + last_refresh_type = %ld \ + WHERE refresh_mode = %ld and \ + last_refresh_scn < %lu AND last_refresh_scn > 0", + OB_ALL_MVIEW_TNAME, scn_val, last_refresh_type, + ObMVRefreshMode::MAJOR_COMPACTION, + scn_val))) { + LOG_WARN("fail to assign sql", KR(ret)); + } else if (OB_FAIL(sql_client.write(tenant_id, sql.ptr(), affected_rows))) { + LOG_WARN("execute sql failed", KR(ret), K(sql)); + } else if (OB_UNLIKELY(affected_rows < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected affected_rows", K(ret), K(affected_rows)); + } + } + + return ret; +} + +int ObMViewInfo::get_min_major_refresh_mview_scn(ObISQLClient &sql_client, const uint64_t tenant_id, + int64_t snapshot_for_tx, share::SCN &scn) +{ + int ret = OB_SUCCESS; + scn.reset(); + ObSqlString sql; + + SMART_VAR(ObMySQLProxy::MySQLResult, res) + { + ObMySQLResult *result = nullptr; + if (OB_FAIL(sql.assign_fmt( + "SELECT min(last_refresh_scn) min_refresh_scn FROM %s as of snapshot %ld WHERE " + "refresh_mode = %ld ", + OB_ALL_MVIEW_TNAME, snapshot_for_tx, ObMVRefreshMode::MAJOR_COMPACTION))) { + LOG_WARN("fail to assign sql", KR(ret)); + } else if (OB_FAIL(sql_client.read(res, tenant_id, sql.ptr()))) { + LOG_WARN("execute sql failed", KR(ret), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is null", KR(ret)); + } else if (OB_FAIL(result->next())) { + LOG_WARN("fail to get next", KR(ret)); + } else { + uint64_t scn_val = 0; + EXTRACT_UINT_FIELD_MYSQL(*result, "min_refresh_scn", scn_val, uint64_t); + if (OB_SUCC(ret)) { + scn.convert_for_inner_table_field(scn_val); + } + } + } + + return ret; +} + +int ObMViewInfo::contains_major_refresh_mview_in_creation(ObISQLClient &sql_client, + const uint64_t tenant_id, bool &contains) +{ + int ret = OB_SUCCESS; + contains = false; + ObSqlString sql; + + SMART_VAR(ObMySQLProxy::MySQLResult, res) + { + ObMySQLResult *result = nullptr; + if (OB_FAIL(sql.assign_fmt( + "SELECT count(*) cnt FROM %s WHERE refresh_mode = %ld and last_refresh_scn = 0", + OB_ALL_MVIEW_TNAME, ObMVRefreshMode::MAJOR_COMPACTION))) { + LOG_WARN("fail to assign sql", KR(ret)); + } else if (OB_FAIL(sql_client.read(res, tenant_id, sql.ptr()))) { + LOG_WARN("execute sql failed", KR(ret), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is null", KR(ret)); + } else if (OB_FAIL(result->next())) { + LOG_WARN("fail to get next", KR(ret)); + } else { + int64_t cnt = 0; + EXTRACT_INT_FIELD_MYSQL(*result, "cnt", cnt, int64_t); + if (OB_SUCC(ret) && cnt > 0) { + contains = true; + } + } + } + + return ret; +} } // namespace schema } // namespace share } // namespace oceanbase diff --git a/src/share/schema/ob_mview_info.h b/src/share/schema/ob_mview_info.h index d9ee9d808c..09a3f3b643 100644 --- a/src/share/schema/ob_mview_info.h +++ b/src/share/schema/ob_mview_info.h @@ -65,6 +65,12 @@ public: #undef DEFINE_GETTER_AND_SETTER #undef DEFINE_STRING_GETTER_AND_SETTER + bool is_fast_lsm_mv() const + { + return (refresh_method_ == ObMVRefreshMethod::FAST && + refresh_mode_ == ObMVRefreshMode::MAJOR_COMPACTION); + } + int gen_insert_mview_dml(const uint64_t exec_tenant_id, ObDMLSqlSplicer &dml) const; int gen_update_mview_attribute_dml(const uint64_t exec_tenant_id, ObDMLSqlSplicer &dml) const; int gen_update_mview_last_refresh_info_dml(const uint64_t exec_tenant_id, @@ -83,7 +89,12 @@ public: static int batch_fetch_mview_ids(ObISQLClient &sql_client, uint64_t tenant_id, uint64_t last_mview_id, ObIArray &mview_ids, int64_t limit = -1); - + static int update_major_refresh_mview_scn(ObISQLClient &sql_client, const uint64_t tenant_id, + const share::SCN &scn); + static int get_min_major_refresh_mview_scn(ObISQLClient &sql_client, const uint64_t tenant_id, + int64_t snapshot_for_tx, share::SCN &scn); + static int contains_major_refresh_mview_in_creation(ObISQLClient &sql_client, + const uint64_t tenant_id, bool &contains); TO_STRING_KV(K_(tenant_id), K_(mview_id), K_(build_mode), diff --git a/src/share/schema/ob_schema_printer.cpp b/src/share/schema/ob_schema_printer.cpp index 4e26c1b56a..c13bc304d7 100644 --- a/src/share/schema/ob_schema_printer.cpp +++ b/src/share/schema/ob_schema_printer.cpp @@ -1751,8 +1751,12 @@ int ObSchemaPrinter::print_table_definition_table_options(const ObTableSchema &t bool is_oracle_mode = false; const bool is_index_tbl = table_schema.is_index_table(); const uint64_t tenant_id = table_schema.get_tenant_id(); + uint64_t data_version = OB_INVALID_VERSION; + if (OB_FAIL(table_schema.check_if_oracle_compat_mode(is_oracle_mode))) { LOG_WARN("fail to check oracle mode", KR(ret), K(table_schema)); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { + LOG_WARN("get min data_version failed", K(ret), K(tenant_id)); } if (OB_SUCCESS == ret && !table_schema.is_external_table() && !is_index_tbl && !is_for_table_status && !is_no_field_options(sql_mode) && !is_no_table_options(sql_mode)) { @@ -1982,6 +1986,13 @@ int ObSchemaPrinter::print_table_definition_table_options(const ObTableSchema &t if (table_schema.get_duplicate_scope() == ObDuplicateScope::DUPLICATE_SCOPE_CLUSTER) { if (OB_FAIL(databuff_printf(buf, buf_len, pos, "DUPLICATE_SCOPE = 'CLUSTER' "))) { SHARE_SCHEMA_LOG(WARN, "fail to print table duplicate scope", K(ret)); + } else if (data_version < DATA_VERSION_4_3_4_0) { + SHARE_SCHEMA_LOG(INFO, "data version is less than 4.3.4, not show duplicate_read_consistency option", K(ret)); + } else if (ObDuplicateReadConsistencyChecker::is_valid_duplicate_read_consistency(table_schema.get_duplicate_read_consistency())) { + if (OB_FAIL(databuff_printf(buf, buf_len, pos, "DUPLICATE_READ_CONSISTENCY = '%s' ", + ObDuplicateReadConsistencyChecker::get_duplicate_read_consistency_str(table_schema.get_duplicate_read_consistency())))) { + SHARE_SCHEMA_LOG(WARN, "fail to print table duplicate read consistency", K(ret)); + } } } } diff --git a/src/share/schema/ob_schema_retrieve_utils.ipp b/src/share/schema/ob_schema_retrieve_utils.ipp index 675f701b9f..23a1a3f622 100644 --- a/src/share/schema/ob_schema_retrieve_utils.ipp +++ b/src/share/schema/ob_schema_retrieve_utils.ipp @@ -1373,6 +1373,7 @@ int ObSchemaRetrieveUtils::fill_table_schema( EXTRACT_INT_FIELD_TO_CLASS_MYSQL_WITH_DEFAULT_VALUE(result, progressive_merge_round, table_schema, int64_t, true, ObSchemaService::g_ignore_column_retrieve_error_, 0); EXTRACT_INT_FIELD_TO_CLASS_MYSQL_WITH_DEFAULT_VALUE(result, storage_format_version, table_schema, int64_t, true, ObSchemaService::g_ignore_column_retrieve_error_, 0); EXTRACT_INT_FIELD_TO_CLASS_MYSQL_WITH_DEFAULT_VALUE(result, table_mode, table_schema, int32_t, true, ObSchemaService::g_ignore_column_retrieve_error_, 0); + EXTRACT_INT_FIELD_TO_CLASS_MYSQL_WITH_DEFAULT_VALUE(result, mv_mode, table_schema, int64_t, true, true, 0); if (OB_SUCC(ret)) { if (OB_FAIL(table_schema.set_expire_info(expire_info))) { SHARE_SCHEMA_LOG(WARN, "set expire info failed", K(ret)); @@ -1405,9 +1406,18 @@ int ObSchemaRetrieveUtils::fill_table_schema( SHARE_SCHEMA_LOG(WARN, "set part expr failed", K(ret)); } } - + //duplicate attribute const ObDuplicateScope duplicate_scope_default = ObDuplicateScope::DUPLICATE_SCOPE_NONE; - EXTRACT_INT_FIELD_TO_CLASS_MYSQL_WITH_DEFAULT_VALUE(result, duplicate_scope, table_schema, int64_t, true, ObSchemaService::g_ignore_column_retrieve_error_, duplicate_scope_default); + const ObDuplicateReadConsistency duplicate_read_consistency_default = ObDuplicateReadConsistency::STRONG; + ObDuplicateScope duplicate_scope = duplicate_scope_default; + ObDuplicateReadConsistency duplicate_read_consistency = duplicate_read_consistency_default; + EXTRACT_INT_FIELD_MYSQL_WITH_DEFAULT_VALUE(result, "duplicate_scope", duplicate_scope, ObDuplicateScope, true /* skip null error*/, + ObSchemaService::g_ignore_column_retrieve_error_, duplicate_scope_default); + EXTRACT_INT_FIELD_MYSQL_WITH_DEFAULT_VALUE(result, "duplicate_read_consistency", duplicate_read_consistency, ObDuplicateReadConsistency, true /* skip null error*/, + true, duplicate_read_consistency_default); + if (OB_SUCC(ret)) { + table_schema.set_duplicate_attribute(duplicate_scope, duplicate_read_consistency); + } //encrypt ObString encryption_default(""); ObString encryption; @@ -4457,10 +4467,18 @@ int ObSchemaRetrieveUtils::fill_table_schema( } if (OB_SUCC(ret)) { EXTRACT_VARCHAR_FIELD_TO_CLASS_MYSQL(result, table_name, table_schema); + // duplicate attribute const ObDuplicateScope duplicate_scope_default = ObDuplicateScope::DUPLICATE_SCOPE_NONE; - EXTRACT_INT_FIELD_TO_CLASS_MYSQL_WITH_DEFAULT_VALUE( - result, duplicate_scope, table_schema, int64_t, true /* skip null error*/, - ObSchemaService::g_ignore_column_retrieve_error_, duplicate_scope_default); + const ObDuplicateReadConsistency duplicate_read_consistency_default = ObDuplicateReadConsistency::STRONG; + ObDuplicateScope duplicate_scope = duplicate_scope_default; + ObDuplicateReadConsistency duplicate_read_consistency = duplicate_read_consistency_default; + EXTRACT_INT_FIELD_MYSQL_WITH_DEFAULT_VALUE(result, "duplicate_scope", duplicate_scope, ObDuplicateScope, true /* skip null error*/, + ObSchemaService::g_ignore_column_retrieve_error_, duplicate_scope_default); + EXTRACT_INT_FIELD_MYSQL_WITH_DEFAULT_VALUE(result, "duplicate_read_consistency", duplicate_read_consistency, ObDuplicateReadConsistency, true /* skip null error*/, + true, duplicate_read_consistency_default); + if (OB_SUCC(ret)) { + table_schema.set_duplicate_attribute(duplicate_scope, duplicate_read_consistency); + } ObString encryption_default(""); ObString encryption; EXTRACT_VARCHAR_FIELD_MYSQL_WITH_DEFAULT_VALUE( diff --git a/src/share/schema/ob_schema_service.cpp b/src/share/schema/ob_schema_service.cpp index 338e1e1d1b..eed8876b65 100644 --- a/src/share/schema/ob_schema_service.cpp +++ b/src/share/schema/ob_schema_service.cpp @@ -544,6 +544,9 @@ int AlterTableSchema::assign(const ObTableSchema &src_schema) if (OB_SUCC(ret) && OB_FAIL(deep_copy_str(src_schema.kv_attributes_, kv_attributes_))) { LOG_WARN("Fail to deep copy ttl definition string", K(ret)); } + if (FAILEDx(mv_mode_.assign(src_schema.mv_mode_))) { + LOG_WARN("fail to assign mv_mode", K(ret)); + } if (OB_SUCC(ret) && OB_FAIL(deep_copy_str(src_schema.index_params_, index_params_))) { LOG_WARN("Fail to deep copy vector index param string", K(ret)); } diff --git a/src/share/schema/ob_schema_struct.cpp b/src/share/schema/ob_schema_struct.cpp index e377e8d520..349b9efb8e 100644 --- a/src/share/schema/ob_schema_struct.cpp +++ b/src/share/schema/ob_schema_struct.cpp @@ -4747,16 +4747,6 @@ int ObTablegroupSchema::get_zone_list( return ret; } -int ObTablegroupSchema::check_is_duplicated( - share::schema::ObSchemaGetterGuard &guard, - bool &is_duplicated) const -{ - int ret = OB_SUCCESS; - UNUSED(guard); - is_duplicated = false; - return ret; -} - //TODO: These functions can be extracted; as an implementation of locality/primary_zone int ObTablegroupSchema::get_zone_replica_attr_array_inherit( ObSchemaGetterGuard &schema_guard, diff --git a/src/share/schema/ob_schema_struct.h b/src/share/schema/ob_schema_struct.h index c5e385ed39..953cb8f464 100755 --- a/src/share/schema/ob_schema_struct.h +++ b/src/share/schema/ob_schema_struct.h @@ -2524,17 +2524,12 @@ public: share::schema::ObSchemaGetterGuard &schema_guard, const common::ObIArray &replica_addrs, common::ObZone &first_primary_zone) const = 0; - virtual int check_is_duplicated( - share::schema::ObSchemaGetterGuard &guard, - bool &is_duplicated) const = 0; virtual uint64_t get_tablegroup_id() const = 0; virtual void set_tablegroup_id(const uint64_t tg_id) = 0; virtual int get_paxos_replica_num( share::schema::ObSchemaGetterGuard &schema_guard, int64_t &num) const = 0; virtual share::ObDuplicateScope get_duplicate_scope() const = 0; - virtual void set_duplicate_scope(const share::ObDuplicateScope duplicate_scope) = 0; - virtual void set_duplicate_scope(const int64_t duplicate_scope) = 0; inline virtual int64_t get_part_func_expr_num() const { return 0; } inline virtual void set_part_func_expr_num(const int64_t part_func_expr_num) { UNUSED(part_func_expr_num); } inline virtual int64_t get_sub_part_func_expr_num() const { return 0; } @@ -2922,9 +2917,6 @@ public: share::schema::ObSchemaGetterGuard &schema_guard, const common::ObIArray &replica_addrs, common::ObZone &first_primary_zone) const override; - virtual int check_is_duplicated( - share::schema::ObSchemaGetterGuard &guard, - bool &is_duplicated) const override; virtual int get_primary_zone_inherit( share::schema::ObSchemaGetterGuard &schema_guard, share::schema::ObPrimaryZone &primary_zone) const override; @@ -2944,8 +2936,6 @@ public: int64_t &num) const; //partition related virtual share::ObDuplicateScope get_duplicate_scope() const override { return share::ObDuplicateScope::DUPLICATE_SCOPE_NONE; } - virtual void set_duplicate_scope(const share::ObDuplicateScope duplicate_scope) override { UNUSED(duplicate_scope); } - virtual void set_duplicate_scope(const int64_t duplicate_scope) override { UNUSED(duplicate_scope); } inline virtual bool is_user_partition_table() const override { return PARTITION_LEVEL_ONE == get_part_level() @@ -3540,6 +3530,7 @@ enum struct ObMVRefreshMode : int64_t DEMAND = 1, COMMIT = 2, STATEMENT = 3, + MAJOR_COMPACTION = 4, MAX }; diff --git a/src/share/schema/ob_table_param.cpp b/src/share/schema/ob_table_param.cpp index 01988b80cd..bc0b3328a6 100644 --- a/src/share/schema/ob_table_param.cpp +++ b/src/share/schema/ob_table_param.cpp @@ -1040,10 +1040,14 @@ int ObTableParam::construct_columns_and_projector( } else if (OB_UNLIKELY(common::OB_HIDDEN_TRANS_VERSION_COLUMN_ID == column_id) || common::OB_HIDDEN_SQL_SEQUENCE_COLUMN_ID == column_id || common::OB_HIDDEN_LOGICAL_ROWID_COLUMN_ID == column_id || + common::OB_MAJOR_REFRESH_MVIEW_OLD_NEW_COLUMN_ID == column_id || common::OB_HIDDEN_GROUP_IDX_COLUMN_ID == column_id) { ObObjMeta meta_type; if (common::OB_HIDDEN_LOGICAL_ROWID_COLUMN_ID == column_id) { meta_type.set_urowid(); + } else if (common::OB_MAJOR_REFRESH_MVIEW_OLD_NEW_COLUMN_ID == column_id) { + meta_type.set_varchar(); + meta_type.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_GENERAL_CI); } else { meta_type.set_int(); } diff --git a/src/share/schema/ob_table_schema.cpp b/src/share/schema/ob_table_schema.cpp index aa89b49d83..8f81578479 100644 --- a/src/share/schema/ob_table_schema.cpp +++ b/src/share/schema/ob_table_schema.cpp @@ -120,6 +120,31 @@ bool ObTableMode::is_valid() const OB_SERIALIZE_MEMBER_SIMPLE(ObTableMode, mode_); +bool ObMvMode::is_valid() const +{ + bool bret = true; + // reserve for new flags + return bret; +} + +int ObMvMode::assign(const ObMvMode &other) +{ + int ret = OB_SUCCESS; + mode_ = other.mode_; + return ret; +} + +ObMvMode & ObMvMode::operator=(const ObMvMode &other) +{ + if (this != &other) { + mode_ = other.mode_; + } + return *this; +} + +OB_SERIALIZE_MEMBER_SIMPLE(ObMvMode, + mode_); + common::ObString ObMergeSchema::EMPTY_STRING = common::ObString::make_string(""); int ObMergeSchema::get_mulit_version_rowkey_column_ids(common::ObIArray &column_ids) const @@ -1607,6 +1632,10 @@ int ObTableSchema::assign(const ObTableSchema &src_schema) LOG_WARN("fail to assign depend_mock_fk_parent_table_ids_ array", K(ret)); } + if (FAILEDx(mv_mode_.assign(src_schema.mv_mode_))) { + LOG_WARN("fail to assign mv_mode", K(ret)); + } + //copy columns column_cnt = src_schema.column_cnt_; // copy constraints @@ -3623,6 +3652,7 @@ void ObTableSchema::reset() cg_name_hash_arr_ = NULL; mlog_tid_ = OB_INVALID_ID; local_session_vars_.reset(); + mv_mode_.reset(); ObSimpleTableSchemaV2::reset(); } @@ -6585,24 +6615,6 @@ int ObSimpleTableSchemaV2::check_if_oracle_compat_mode(bool &is_oracle_mode) con return ObCompatModeGetter::check_is_oracle_mode_with_table_id(tenant_id, table_id, is_oracle_mode); } -int ObSimpleTableSchemaV2::check_is_duplicated( - share::schema::ObSchemaGetterGuard &guard, - bool &is_duplicated) const -{ - int ret = OB_SUCCESS; - const uint64_t tenant_id = get_tenant_id(); - bool is_restore = false; - is_duplicated = false; - if (OB_FAIL(guard.check_tenant_is_restore(tenant_id, is_restore))) { - LOG_WARN("fail to check tenant is restore", K(ret), K(tenant_id)); - } else if (is_restore) { - is_duplicated = false; - } else if (ObDuplicateScope::DUPLICATE_SCOPE_CLUSTER == get_duplicate_scope()) { - is_duplicated = true; - } - return ret; -} - int ObTableSchema::get_generated_column_by_define(const ObString &col_def, const bool only_hidden_column, ObColumnSchemaV2 *&gen_col) @@ -6985,6 +6997,7 @@ OB_DEF_SERIALIZE(ObTableSchema) OB_UNIS_ENCODE(index_params_); } OB_UNIS_ENCODE(micro_index_clustered_); + OB_UNIS_ENCODE(mv_mode_); return ret; } @@ -7441,6 +7454,7 @@ OB_DEF_DESERIALIZE(ObTableSchema) } } OB_UNIS_DECODE(micro_index_clustered_); + OB_UNIS_DECODE(mv_mode_); return ret; } @@ -7596,6 +7610,7 @@ OB_DEF_SERIALIZE_SIZE(ObTableSchema) OB_UNIS_ADD_LEN(duplicate_read_consistency_); OB_UNIS_ADD_LEN(index_params_); OB_UNIS_ADD_LEN(micro_index_clustered_); + OB_UNIS_ADD_LEN(mv_mode_); return len; } diff --git a/src/share/schema/ob_table_schema.h b/src/share/schema/ob_table_schema.h index bae28cc7b9..04a0c5421d 100644 --- a/src/share/schema/ob_table_schema.h +++ b/src/share/schema/ob_table_schema.h @@ -291,6 +291,18 @@ enum ObDDLIgnoreSyncCdcFlag DONT_SYNC_LOG_FOR_CDC = 1, }; +enum ObMVMajorRefreshFlag +{ + IS_NOT_MV_MAJOR_REFRESH = 0, + IS_MV_MAJOR_REFRESH = 1 +}; + +enum ObTableReferencedByFastLSMMVFlag +{ + IS_NOT_REFERENCED_BY_FAST_LSM_MV = 0, + IS_REFERENCED_BY_FAST_LSM_MV = 1 +}; + struct ObTableMode { OB_UNIS_VERSION_V(1); private: @@ -322,9 +334,7 @@ private: static const int32_t TM_MV_ON_QUERY_COMPUTATION_BITS = 1; static const int32_t TM_DDL_IGNORE_SYNC_CDC_OFFSET = 29; static const int32_t TM_DDL_IGNORE_SYNC_CDC_BITS = 1; - static const int32_t TM_MV_MAJOR_REFRESH_OFFSET = 30; - static const int32_t TM_MV_MAJOR_REFRESH_BITS = 1; - static const int32_t TM_RESERVED = 1; + static const int32_t TM_RESERVED = 2; static const uint32_t MODE_FLAG_MASK = (1U << TM_MODE_FLAG_BITS) - 1; static const uint32_t PK_MODE_MASK = (1U << TM_PK_MODE_BITS) - 1; @@ -437,11 +447,55 @@ public: uint32_t mv_enable_query_rewrite_flag_ : TM_MV_ENABLE_QUERY_REWRITE_BITS; uint32_t mv_on_query_computation_flag_ : TM_MV_ON_QUERY_COMPUTATION_BITS; uint32_t ddl_table_ignore_sync_cdc_flag_ : TM_DDL_IGNORE_SYNC_CDC_BITS; - uint32_t reserved_ :TM_RESERVED; + uint32_t reserved_ : TM_RESERVED; }; }; }; +struct ObMvMode { + OB_UNIS_VERSION_V(1); +private: + static const int32_t MM_MV_MAJOR_REFRESH_OFFSET = 0; + static const int32_t MM_MV_MAJOR_REFRESH_BITS = 1; + static const uint32_t MM_MV_MAJOR_REFRESH_MASK = (1U << MM_MV_MAJOR_REFRESH_BITS) - 1; + static const int32_t MM_TABLE_REFERENCED_BY_FAST_LSM_MV_OFFSET = 1; + static const int32_t MM_TABLE_REFERENCED_BY_FAST_LSM_MV_BITS = 1; + static const uint32_t MM_TABLE_REFERENCED_BY_FAST_LSM_MV_MASK = + (1U << MM_TABLE_REFERENCED_BY_FAST_LSM_MV_BITS) - 1; + static const int32_t MM_RESERVED = 62; +public: + ObMvMode() { reset(); } + virtual ~ObMvMode() { reset(); } + void reset() { mode_ = 0; } + bool operator==(const ObMvMode &other) const { return mode_ == other.mode_; } + int assign(const ObMvMode &other); + ObMvMode &operator=(const ObMvMode &other); + bool is_valid() const; + static ObMVMajorRefreshFlag get_mv_major_refresh_flag(int64_t mv_mode) + { + return (ObMVMajorRefreshFlag)((mv_mode >> MM_MV_MAJOR_REFRESH_OFFSET) & + MM_MV_MAJOR_REFRESH_MASK); + } + static ObTableReferencedByFastLSMMVFlag get_table_referenced_by_fast_lsm_mv_flag(int64_t mv_mode) + { + return (ObTableReferencedByFastLSMMVFlag)( + (mv_mode >> MM_TABLE_REFERENCED_BY_FAST_LSM_MV_OFFSET) & + MM_TABLE_REFERENCED_BY_FAST_LSM_MV_MASK); + } + union + { + int64_t mode_; + struct + { + uint64_t mv_major_refresh_flag_ : MM_MV_MAJOR_REFRESH_BITS; + uint64_t table_referenced_by_fast_lsm_mv_flag_ : MM_TABLE_REFERENCED_BY_FAST_LSM_MV_BITS; + uint64_t reserved_ : MM_RESERVED; + }; + }; + TO_STRING_KV("mv_major_refresh_flag", mv_major_refresh_flag_, + "table_referenced_by_fast_lsm_mv_flag", table_referenced_by_fast_lsm_mv_flag_); +}; + struct ObBackUpTableModeOp { /* @@ -652,6 +706,11 @@ public: UNUSED(skip_idx_attrs); return common::OB_NOT_SUPPORTED; } + virtual int get_mv_mode_struct(ObMvMode &mv_mode) const + { + UNUSED(mv_mode); + return common::OB_NOT_SUPPORTED; + } DECLARE_PURE_VIRTUAL_TO_STRING; const static int64_t INVAID_RET = -1; static common::ObString EMPTY_STRING; @@ -816,9 +875,6 @@ public: virtual int get_paxos_replica_num( share::schema::ObSchemaGetterGuard &guard, int64_t &num) const override; - virtual int check_is_duplicated( - share::schema::ObSchemaGetterGuard &guard, - bool &is_duplicated) const override; virtual int get_first_primary_zone_inherit( share::schema::ObSchemaGetterGuard &schema_guard, const common::ObIArray &replica_addrs, @@ -1047,12 +1103,22 @@ public: inline bool is_partitioned_table() const { return PARTITION_LEVEL_ONE == get_part_level() || PARTITION_LEVEL_TWO == get_part_level(); } virtual ObPartitionLevel get_part_level() const override; virtual share::ObDuplicateScope get_duplicate_scope() const override { return duplicate_scope_; } - inline bool is_duplicate_table() const { return duplicate_scope_ != ObDuplicateScope::DUPLICATE_SCOPE_NONE; } - virtual void set_duplicate_scope(const share::ObDuplicateScope duplicate_scope) override { duplicate_scope_ = duplicate_scope; } - virtual void set_duplicate_scope(const int64_t duplicate_scope) override { duplicate_scope_ = static_cast(duplicate_scope); } - + inline void set_duplicate_attribute(const share::ObDuplicateScope duplicate_scope, + const share::ObDuplicateReadConsistency duplicate_read_consistency) { + duplicate_scope_ = duplicate_scope; + duplicate_read_consistency_ = duplicate_read_consistency; + } inline void set_duplicate_read_consistency(const share::ObDuplicateReadConsistency duplicate_read_consistency) { duplicate_read_consistency_ = duplicate_read_consistency; } inline share::ObDuplicateReadConsistency get_duplicate_read_consistency() const { return duplicate_read_consistency_; } + + inline bool is_duplicate_table() const { + return duplicate_scope_ == ObDuplicateScope::DUPLICATE_SCOPE_CLUSTER + && duplicate_read_consistency_ == ObDuplicateReadConsistency::STRONG; + } + inline bool is_broadcast_table() const { + return duplicate_scope_ == ObDuplicateScope::DUPLICATE_SCOPE_CLUSTER + && duplicate_read_consistency_ == ObDuplicateReadConsistency::WEAK; + } // for encrypt int set_encryption_str(const common::ObString &str) { return deep_copy_str(str, encryption_); } virtual const common::ObString &get_encryption_str() const override { return encryption_; } @@ -1845,6 +1911,7 @@ public: int64_t get_lob_columns_count() const; bool has_lob_aux_table() const { return (aux_lob_meta_tid_ != OB_INVALID_ID && aux_lob_piece_tid_ != OB_INVALID_ID); } bool has_mlog_table() const { return (OB_INVALID_ID != mlog_tid_); } + bool required_by_mview_refresh() const { return has_mlog_table() || table_referenced_by_fast_lsm_mv(); } inline void add_table_flag(uint64_t flag) { table_flags_ |= flag; } inline void del_table_flag(uint64_t flag) { table_flags_ &= ~flag; } inline void add_or_del_table_flag(uint64_t flag, bool is_add) @@ -1875,6 +1942,30 @@ public: uint64_t get_mlog_tid() const { return mlog_tid_; } inline sql::ObLocalSessionVar &get_local_session_var() { return local_session_vars_; } inline const sql::ObLocalSessionVar &get_local_session_var() const { return local_session_vars_; } + inline void set_mv_mode(const int64_t mv_mode) { mv_mode_.mode_ = mv_mode; } + inline int64_t get_mv_mode() const { return mv_mode_.mode_; } + virtual int get_mv_mode_struct(ObMvMode &mv_mode) const override + { + mv_mode = mv_mode_; + return OB_SUCCESS; + } + inline bool mv_major_refresh() const + { + return IS_MV_MAJOR_REFRESH == (enum ObMVMajorRefreshFlag)mv_mode_.mv_major_refresh_flag_; + } + inline void set_mv_major_refresh(const ObMVMajorRefreshFlag flag) + { + mv_mode_.mv_major_refresh_flag_ = flag; + } + inline bool table_referenced_by_fast_lsm_mv() const + { + return IS_REFERENCED_BY_FAST_LSM_MV == + (enum ObTableReferencedByFastLSMMVFlag)mv_mode_.table_referenced_by_fast_lsm_mv_flag_; + } + inline void set_table_referenced_by_fast_lsm_mv(const ObTableReferencedByFastLSMMVFlag flag) + { + mv_mode_.table_referenced_by_fast_lsm_mv_flag_ = flag; + } DECLARE_VIRTUAL_TO_STRING; protected: @@ -2084,6 +2175,7 @@ protected: common::ObString index_params_; // exec_env common::ObString exec_env_; + ObMvMode mv_mode_; }; class ObPrintableTableSchema final : public ObTableSchema diff --git a/src/share/schema/ob_table_sql_service.cpp b/src/share/schema/ob_table_sql_service.cpp index f0449e236c..19e3f25c8c 100644 --- a/src/share/schema/ob_table_sql_service.cpp +++ b/src/share/schema/ob_table_sql_service.cpp @@ -2822,13 +2822,19 @@ int ObTableSqlService::update_mview_reference_table_status( share::schema::ObTableMode table_mode_struct = table_schema.get_table_mode_struct(); uint64_t table_id = table_schema.get_table_id(); int64_t new_schema_version = table_schema.get_schema_version(); + int64_t mv_mode = table_schema.get_mv_mode(); ObDMLSqlSplicer dml; - if (OB_FAIL(dml.add_pk_column("tenant_id", ObSchemaUtils::get_extract_tenant_id( - exec_tenant_id, tenant_id))) - || OB_FAIL(dml.add_pk_column("table_id", ObSchemaUtils::get_extract_schema_id( - exec_tenant_id, table_id))) - || OB_FAIL(dml.add_column("schema_version", new_schema_version)) - || OB_FAIL(dml.add_column("table_mode", table_mode_struct.mode_))) { + uint64_t data_version = 0; + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { + LOG_WARN("failed to get data version", KR(ret)); + } else if (OB_FAIL(dml.add_pk_column( + "tenant_id", ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, tenant_id))) || + OB_FAIL(dml.add_pk_column( + "table_id", ObSchemaUtils::get_extract_schema_id(exec_tenant_id, table_id))) || + OB_FAIL(dml.add_column("schema_version", new_schema_version)) || + OB_FAIL(dml.add_column("table_mode", table_mode_struct.mode_)) || + (data_version >= DATA_VERSION_4_3_4_0 && + OB_FAIL(dml.add_column("mv_mode", mv_mode)))) { LOG_WARN("failed to add column", KR(ret), K(exec_tenant_id), K(tenant_id)); } else { int64_t affected_rows = 0; @@ -2856,6 +2862,7 @@ int ObTableSqlService::update_mview_reference_table_status( } else { const bool only_history = true; new_schema.set_table_referenced_by_mv(ObTableMode::get_table_referenced_by_mv_flag(table_mode_struct.mode_)); + new_schema.set_mv_mode(mv_mode); new_schema.set_schema_version(new_schema_version); new_schema.set_in_offline_ddl_white_list(table_schema.get_in_offline_ddl_white_list()); if (OB_FAIL(add_table(sql_client, @@ -3243,6 +3250,10 @@ int ObTableSqlService::gen_table_dml( && OB_FAIL(dml.add_column("column_store", table.is_column_store_supported()))) || ((data_version >= DATA_VERSION_4_3_2_0 || (data_version < DATA_VERSION_4_3_0_0 && data_version >= MOCK_DATA_VERSION_4_2_3_0)) && OB_FAIL(dml.add_column("auto_increment_cache_size", table.get_auto_increment_cache_size()))) + || ((data_version >= DATA_VERSION_4_3_4_0) + && OB_FAIL(dml.add_column("duplicate_read_consistency", table.get_duplicate_read_consistency()))) + || ((data_version >= DATA_VERSION_4_3_4_0) + && OB_FAIL(dml.add_column("mv_mode", table.get_mv_mode()))) || (data_version >= DATA_VERSION_4_3_2_1 && OB_FAIL(dml.add_column("external_properties", ObHexEscapeSqlStr(table.get_external_properties())))) || (data_version >= DATA_VERSION_4_3_3_0 @@ -3412,6 +3423,10 @@ int ObTableSqlService::gen_table_options_dml( && OB_FAIL(dml.add_column("column_store", table.is_column_store_supported()))) || ((data_version >= DATA_VERSION_4_3_2_0 || (data_version < DATA_VERSION_4_3_0_0 && data_version >= MOCK_DATA_VERSION_4_2_3_0)) && OB_FAIL(dml.add_column("auto_increment_cache_size", table.get_auto_increment_cache_size()))) + || ((data_version >= DATA_VERSION_4_3_4_0) + && OB_FAIL(dml.add_column("duplicate_read_consistency", table.get_duplicate_read_consistency()))) + || ((data_version >= DATA_VERSION_4_3_4_0) + && OB_FAIL(dml.add_column("mv_mode", table.get_mv_mode()))) || (data_version >= DATA_VERSION_4_3_3_0 && OB_FAIL(dml.add_column("index_params", ObHexEscapeSqlStr(index_params)))) ) { diff --git a/src/sql/code_generator/ob_dml_cg_service.cpp b/src/sql/code_generator/ob_dml_cg_service.cpp index 487de019ec..08f2fb7096 100644 --- a/src/sql/code_generator/ob_dml_cg_service.cpp +++ b/src/sql/code_generator/ob_dml_cg_service.cpp @@ -3443,7 +3443,7 @@ int ObDmlCgService::generate_table_loc_meta(const IndexDMLInfo &index_dml_info, loc_meta.table_loc_id_ = index_dml_info.loc_table_id_; loc_meta.ref_table_id_ = index_dml_info.ref_table_id_; loc_meta.select_leader_ = 1; - loc_meta.is_dup_table_ = (ObDuplicateScope::DUPLICATE_SCOPE_NONE != table_schema->get_duplicate_scope()); + loc_meta.is_dup_table_ = table_schema->is_duplicate_table(); //related local index tablet_id pruning only can be used in local plan or remote plan(all operator //use the same das context), //because the distributed plan will transfer tablet_id through exchange operator, @@ -3776,7 +3776,7 @@ int ObDmlCgService::generate_fk_table_loc_info(uint64_t index_table_id, loc_meta.table_loc_id_ = index_table_id; loc_meta.ref_table_id_ = index_table_id; loc_meta.select_leader_ = 1; - loc_meta.is_dup_table_ = (ObDuplicateScope::DUPLICATE_SCOPE_NONE != table_schema->get_duplicate_scope()); + loc_meta.is_dup_table_ = (table_schema->is_duplicate_table()); if (PARTITION_LEVEL_ZERO == table_schema->get_part_level()) { tablet_id = table_schema->get_tablet_id(); } else { diff --git a/src/sql/code_generator/ob_static_engine_cg.cpp b/src/sql/code_generator/ob_static_engine_cg.cpp index bc1fe0f08f..c916059675 100644 --- a/src/sql/code_generator/ob_static_engine_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_cg.cpp @@ -931,6 +931,7 @@ int ObStaticEngineCG::generate_calc_exprs( && T_PSEUDO_EXTERNAL_FILE_COL != raw_expr->get_expr_type() && T_PSEUDO_EXTERNAL_FILE_URL != raw_expr->get_expr_type() && T_PSEUDO_PARTITION_LIST_COL != raw_expr->get_expr_type() + && T_ORA_ROWSCN != raw_expr->get_expr_type() && !(raw_expr->is_const_expr() || raw_expr->has_flag(IS_DYNAMIC_USER_VARIABLE)) && !(T_FUN_SYS_PART_HASH == raw_expr->get_expr_type() || T_FUN_SYS_PART_KEY == raw_expr->get_expr_type()) && !(raw_expr->is_vector_sort_expr())) { diff --git a/src/sql/code_generator/ob_tsc_cg_service.cpp b/src/sql/code_generator/ob_tsc_cg_service.cpp index ebd0386c01..f60cb26a72 100644 --- a/src/sql/code_generator/ob_tsc_cg_service.cpp +++ b/src/sql/code_generator/ob_tsc_cg_service.cpp @@ -44,8 +44,8 @@ int ObTscCgService::generate_tsc_ctdef(ObLogTableScan &op, ObTableScanCtDef &tsc } else { query_flag.scan_order_ = ObQueryFlag::Forward; } + OZ(generate_mr_mv_scan_flag(op, query_flag)); tsc_ctdef.scan_flags_ = query_flag; - if (op.use_index_merge()) { tsc_ctdef.use_index_merge_ = true; } @@ -127,8 +127,8 @@ int ObTscCgService::generate_tsc_ctdef(ObLogTableScan &op, ObTableScanCtDef &tsc if (OB_ISNULL(info)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid table location info", K(ret)); - } else if (info->get_table_location().use_das() && - info->get_table_location().get_has_dynamic_exec_param()) { + } else if ((info->get_table_location().use_das() && info->get_table_location().get_has_dynamic_exec_param()) + || info->get_table_location().is_dynamic_replica_select_table()) { if (OB_FAIL(tsc_ctdef.allocate_dppr_table_loc())) { LOG_WARN("allocate dppr table location failed", K(ret)); } else if (OB_FAIL(tsc_ctdef.das_dppr_tbl_->assign(info->get_table_location()))) { @@ -574,6 +574,7 @@ int ObTscCgService::generate_tsc_filter(const ObLogTableScan &op, ObTableScanSpe op.get_access_exprs(), op.get_type(), op.get_index_back() && op.get_is_index_global(), + op.use_column_store(), lookup_ctdef->pd_expr_spec_))) { LOG_WARN("generate pd storage flag for lookup ctdef failed", K(ret)); } @@ -588,6 +589,7 @@ int ObTscCgService::generate_tsc_filter(const ObLogTableScan &op, ObTableScanSpe op.get_access_exprs(), op.get_type(), false, /*generate_pd_storage_flag*/ + op.use_column_store(), scan_ctdef.pd_expr_spec_))) { LOG_WARN("generate pd storage flag for scan ctdef failed", K(ret)); } else if (lookup_ctdef != nullptr && @@ -596,6 +598,7 @@ int ObTscCgService::generate_tsc_filter(const ObLogTableScan &op, ObTableScanSpe op.get_access_exprs(), op.get_type(), op.get_index_back() && op.get_is_index_global(), /*generate_pd_storage_flag*/ + op.use_column_store(), lookup_ctdef->pd_expr_spec_))) { LOG_WARN("generate pd storage flag for lookup ctdef failed", K(ret)); } @@ -641,6 +644,7 @@ int ObTscCgService::generate_pd_storage_flag(const ObLogPlan *log_plan, const ObIArray &access_exprs, const log_op_def::ObLogOpType op_type, const bool is_global_index_lookup, + const bool use_column_store, ObPushdownExprSpec &pd_spec) { int ret = OB_SUCCESS; @@ -674,10 +678,11 @@ int ObTscCgService::generate_pd_storage_flag(const ObLogPlan *log_plan, pd_filter = false; } else { FOREACH_CNT_X(e, access_exprs, pd_blockscan || pd_filter) { - if (T_ORA_ROWSCN == (*e)->get_expr_type() || T_PSEUDO_EXTERNAL_FILE_URL == (*e)->get_expr_type()) { + if ((use_column_store && T_ORA_ROWSCN == (*e)->get_expr_type()) || T_PSEUDO_EXTERNAL_FILE_URL == (*e)->get_expr_type() + || T_PSEUDO_OLD_NEW_COL == (*e)->get_expr_type()) { pd_blockscan = false; pd_filter = false; - } else { + } else if (T_ORA_ROWSCN != (*e)->get_expr_type()) { auto col = static_cast(*e); if (col->is_lob_column() && cg_.cur_cluster_version_ < CLUSTER_VERSION_4_1_0_0) { pd_filter = false; @@ -963,6 +968,8 @@ int ObTscCgService::generate_access_ctdef(const ObLogTableScan &op, has_rowscn = true; LOG_DEBUG("need row scn"); } + } else if (T_PSEUDO_OLD_NEW_COL == expr->get_expr_type()) { + OZ(access_column_ids.push_back(OB_MAJOR_REFRESH_MVIEW_OLD_NEW_COLUMN_ID)); } else if (T_PSEUDO_GROUP_ID == expr->get_expr_type()) { OZ(access_column_ids.push_back(common::OB_HIDDEN_GROUP_IDX_COLUMN_ID)); } else if (T_PSEUDO_EXTERNAL_FILE_COL == expr->get_expr_type()) { @@ -1198,6 +1205,7 @@ int ObTscCgService::generate_das_scan_ctdef(const ObLogTableScan &op, op.get_access_exprs(), op.get_type(), op.get_index_back() && op.get_is_index_global(), + op.use_column_store(), scan_ctdef.pd_expr_spec_))) { LOG_WARN("failed to generate pd storage flag for index scan ctdef", K(scan_ctdef.ref_table_id_), K(ret)); } else if (OB_FAIL(cg_.generate_rt_exprs(scan_pushdown_filters, scan_ctdef.pd_expr_spec_.pushdown_filters_))) { @@ -1373,6 +1381,10 @@ int ObTscCgService::extract_das_column_ids(const ObIArray &column_ex if (OB_FAIL(column_ids.push_back(OB_HIDDEN_TRANS_VERSION_COLUMN_ID))) { LOG_WARN("store ora rowscan failed", K(ret)); } + } else if (T_PSEUDO_OLD_NEW_COL == column_exprs.at(i)->get_expr_type()) { + if (OB_FAIL(column_ids.push_back(OB_MAJOR_REFRESH_MVIEW_OLD_NEW_COLUMN_ID))) { + LOG_WARN("store ora rowscan failed", K(ret)); + } } else if (T_PSEUDO_GROUP_ID == column_exprs.at(i)->get_expr_type()) { if (OB_FAIL(column_ids.push_back(OB_HIDDEN_GROUP_IDX_COLUMN_ID))) { LOG_WARN("store group column id failed", K(ret)); @@ -1409,12 +1421,19 @@ int ObTscCgService::generate_table_loc_meta(uint64_t table_loc_id, ObSQLUtils::is_external_files_on_local_disk(table_schema.get_external_file_location()); bool is_weak_read = false; int64_t route_policy = 0; + // broadcast table (insert into select) read local for materialized view create,here three conditions: + // 1. inner sql tag weak read + // 2. is complete refresh + // 3. is broadcast table + const bool is_new_mv_create = ObConsistencyLevel::WEAK == stmt.get_query_ctx()->get_global_hint().read_consistency_ + && table_schema.is_broadcast_table() && session.get_ddl_info().is_mview_complete_refresh(); if (OB_ISNULL(cg_.opt_ctx_) || OB_ISNULL(cg_.opt_ctx_->get_exec_ctx())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(cg_.opt_ctx_), K(ret)); } else if (OB_FAIL(session.get_sys_variable(SYS_VAR_OB_ROUTE_POLICY, route_policy))) { LOG_WARN("get route policy failed", K(ret)); - } else if (stmt.get_query_ctx()->has_dml_write_stmt_) { + } else if (stmt.get_query_ctx()->has_dml_write_stmt_ + && !is_new_mv_create) { loc_meta.select_leader_ = 1; loc_meta.is_weak_read_ = 0; } else if (OB_FAIL(ObTableLocation::get_is_weak_read(stmt, &session, @@ -1424,7 +1443,8 @@ int ObTscCgService::generate_table_loc_meta(uint64_t table_loc_id, } else if (is_weak_read) { loc_meta.is_weak_read_ = 1; loc_meta.select_leader_ = 0; - } else if (loc_meta.is_dup_table_) { + } else if (loc_meta.is_dup_table_ + || is_new_mv_create) { loc_meta.select_leader_ = 0; loc_meta.is_weak_read_ = 0; } else { @@ -3073,5 +3093,29 @@ int ObTscCgService::mapping_oracle_real_agent_virtual_exprs(const ObLogTableScan return ret; } +int ObTscCgService::generate_mr_mv_scan_flag(const ObLogTableScan &op, ObQueryFlag &query_flag) const +{ + int ret = OB_SUCCESS; + const ObLogPlan *log_plan = op.get_plan(); + query_flag.mr_mv_scan_ = op.get_mr_mv_scan(); + if (!query_flag.is_mr_mview_query() && nullptr != log_plan) { + // for query OLD_NEW data from normal table(use hint to mview path) + bool has_enable_param = false; + const ObOptParamHint &opt_params = log_plan->get_stmt()->get_query_ctx()->get_global_hint().opt_params_; + if (OB_FAIL(opt_params.has_opt_param(ObOptParamHint::HIDDEN_COLUMN_VISIBLE, has_enable_param))) { + LOG_WARN("check has hint hidden_column_visible failed", K(ret), K(opt_params)); + } else if (has_enable_param) { + const common::ObIArray &access_exprs = op.get_access_exprs(); + FOREACH_CNT(e, access_exprs) { + if (T_PSEUDO_OLD_NEW_COL == (*e)->get_expr_type()) { + query_flag.mr_mv_scan_ = ObQueryFlag::MRMVScanMode::RealTimeMode; + break; + } + } + } + } + return ret; +} + } // namespace sql } // namespace oceanbase diff --git a/src/sql/code_generator/ob_tsc_cg_service.h b/src/sql/code_generator/ob_tsc_cg_service.h index 8dc25f448c..e48c4f75b1 100644 --- a/src/sql/code_generator/ob_tsc_cg_service.h +++ b/src/sql/code_generator/ob_tsc_cg_service.h @@ -41,6 +41,7 @@ public: const ObIArray &access_exprs, const log_op_def::ObLogOpType op_type, const bool is_global_index_lookup, + const bool use_column_store, ObPushdownExprSpec &pd_spec); int generate_table_loc_meta(uint64_t table_loc_id, const ObDMLStmt &stmt, @@ -154,12 +155,12 @@ private: ObDASSortCtDef *&sort_ctdef); int mapping_oracle_real_agent_virtual_exprs(const ObLogTableScan &op, common::ObIArray &access_exprs); + int generate_mr_mv_scan_flag(const ObLogTableScan &op, ObQueryFlag &query_flag) const; int generate_index_merge_ctdef(const ObLogTableScan &op, ObTableScanCtDef &tsc_ctdef, ObDASBaseCtDef *&root_ctdef); int generate_index_merge_node_ctdef(const ObLogTableScan &op, ObIndexMergeNode *node, common::ObIAllocator &alloc, ObDASBaseCtDef *&node_ctdef); - private: ObStaticEngineCG &cg_; }; diff --git a/src/sql/engine/basic/ob_pushdown_filter.cpp b/src/sql/engine/basic/ob_pushdown_filter.cpp index 53b5a30ee4..ac2c28e750 100644 --- a/src/sql/engine/basic/ob_pushdown_filter.cpp +++ b/src/sql/engine/basic/ob_pushdown_filter.cpp @@ -389,7 +389,7 @@ int ObPushdownFilterConstructor::create_black_filter_node( LOG_WARN("Invalid null raw expr", K(ret)); } else if (OB_FAIL(static_cg_.generate_rt_expr(*raw_expr, expr))) { LOG_WARN("failed to generate rt expr", K(ret)); - } else if (OB_FAIL(ObRawExprUtils::extract_column_exprs(raw_expr, column_exprs))) { + } else if (OB_FAIL(ObRawExprUtils::extract_column_exprs_and_rowscn(raw_expr, column_exprs))) { LOG_WARN("failed to extract column exprs", K(ret)); } else if (OB_FAIL(factory_.alloc(PushdownFilterType::BLACK_FILTER, 0, filter_node))) { LOG_WARN("failed t o alloc pushdown filter", K(ret)); @@ -408,11 +408,19 @@ int ObPushdownFilterConstructor::create_black_filter_node( for (int64_t i = 0; OB_SUCC(ret) && i < column_exprs.count(); ++i) { ObRawExpr *sub_raw_expr = column_exprs.at(i); ObExpr *sub_expr = nullptr; - ObColumnRefRawExpr *ref_expr = static_cast(sub_raw_expr); - if (OB_FAIL(static_cg_.generate_rt_expr(*sub_raw_expr, sub_expr))) { + if (T_ORA_ROWSCN == sub_raw_expr->get_expr_type()) { + if (OB_FAIL(black_filter_node->col_ids_.push_back(common::OB_HIDDEN_TRANS_VERSION_COLUMN_ID))) { + LOG_WARN("failed to push back column id", K(ret)); + } + } else { + ObColumnRefRawExpr *ref_expr = static_cast(sub_raw_expr); + if (OB_FAIL(black_filter_node->col_ids_.push_back(ref_expr->get_column_id()))) { + LOG_WARN("failed to push back column id", K(ret)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(static_cg_.generate_rt_expr(*sub_raw_expr, sub_expr))) { LOG_WARN("failed to generate rt expr", K(ret)); - } else if (OB_FAIL(black_filter_node->col_ids_.push_back(ref_expr->get_column_id()))) { - LOG_WARN("failed to push back column id", K(ret)); } else if (OB_FAIL(black_filter_node->column_exprs_.push_back(sub_expr))) { LOG_WARN("failed to push back column expr", K(ret)); } diff --git a/src/sql/engine/basic/ob_pushdown_filter.h b/src/sql/engine/basic/ob_pushdown_filter.h index 250b77c635..e682938a91 100644 --- a/src/sql/engine/basic/ob_pushdown_filter.h +++ b/src/sql/engine/basic/ob_pushdown_filter.h @@ -662,6 +662,10 @@ public: OB_INLINE void set_filter_rewrited() { is_rewrited_ = true; } OB_INLINE bool is_filter_rewrited() const { return is_rewrited_; } OB_INLINE int64_t get_skipped_rows() const { return skipped_rows_; } + OB_INLINE bool can_pushdown_decoder() + { + return 1 == get_col_count() && common::OB_HIDDEN_TRANS_VERSION_COLUMN_ID != get_col_ids().at(0); + } OB_INLINE void clear_skipped_rows() { skipped_rows_ = 0; } OB_INLINE common::ObIAllocator &get_allocator() { return allocator_; } inline int get_child(uint32_t nth_child, ObPushdownFilterExecutor *&filter_executor) diff --git a/src/sql/engine/expr/ob_expr_last_refresh_scn.cpp b/src/sql/engine/expr/ob_expr_last_refresh_scn.cpp index 73900320eb..5c82a80570 100644 --- a/src/sql/engine/expr/ob_expr_last_refresh_scn.cpp +++ b/src/sql/engine/expr/ob_expr_last_refresh_scn.cpp @@ -185,12 +185,18 @@ int ObExprLastRefreshScn::get_last_refresh_scn_sql(const share::SCN &scn, } if (OB_FAIL(ret)) { } else if (SCN::invalid_scn() == scn) { - if (OB_FAIL(sql.assign_fmt("SELECT MVIEW_ID, LAST_REFRESH_SCN FROM `%s`.`%s` WHERE TENANT_ID = 0 AND MVIEW_ID IN (%.*s)", - OB_SYS_DATABASE_NAME, OB_ALL_MVIEW_TNAME, - (int)mview_id_array.length(), mview_id_array.ptr()))) { + if (OB_FAIL(sql.assign_fmt("SELECT CAST(MVIEW_ID AS UNSIGNED) AS MVIEW_ID, \ + LAST_REFRESH_SCN, \ + CAST(REFRESH_MODE AS UNSIGNED) AS REFRESH_MODE \ + FROM `%s`.`%s` WHERE TENANT_ID = 0 AND MVIEW_ID IN (%.*s)", + OB_SYS_DATABASE_NAME, OB_ALL_MVIEW_TNAME, + (int)mview_id_array.length(), mview_id_array.ptr()))) { LOG_WARN("fail to assign sql", KR(ret)); } - } else if (OB_FAIL(sql.assign_fmt("SELECT MVIEW_ID, LAST_REFRESH_SCN FROM `%s`.`%s` AS OF SNAPSHOT %ld WHERE TENANT_ID = 0 AND MVIEW_ID IN (%.*s)", + } else if (OB_FAIL(sql.assign_fmt("SELECT CAST(MVIEW_ID AS UNSIGNED) AS MVIEW_ID, \ + LAST_REFRESH_SCN, \ + CAST(REFRESH_MODE AS UNSIGNED) AS REFRESH_MODE \ + FROM `%s`.`%s` AS OF SNAPSHOT %ld WHERE TENANT_ID = 0 AND MVIEW_ID IN (%.*s)", OB_SYS_DATABASE_NAME, OB_ALL_MVIEW_TNAME, scn.get_val_for_sql(), (int)mview_id_array.length(), mview_id_array.ptr()))) { LOG_WARN("fail to assign sql", KR(ret), K(scn)); diff --git a/src/sql/engine/table/ob_table_scan_op.cpp b/src/sql/engine/table/ob_table_scan_op.cpp index 0a97f846e8..91fdbb1c58 100644 --- a/src/sql/engine/table/ob_table_scan_op.cpp +++ b/src/sql/engine/table/ob_table_scan_op.cpp @@ -974,16 +974,27 @@ int ObTableScanOp::prepare_das_task() ObSEArray tablet_ids; ObSEArray partition_ids; ObSEArray first_level_part_ids; - if (OB_FAIL(das_location.calculate_tablet_ids(ctx_, - plan_ctx->get_param_store(), - tablet_ids, - partition_ids, - first_level_part_ids, - dtc_params))) { + if (das_location.is_dynamic_replica_select_table() && tsc_rtdef_.dynamic_selected_tablet_id_.is_valid()) { + if (OB_FAIL(tablet_ids.push_back(tsc_rtdef_.dynamic_selected_tablet_id_))) { + LOG_WARN("failed to push back dynamic selected tablet id", K(ret)); + } + } else if (OB_FAIL(das_location.calculate_tablet_ids(ctx_, + plan_ctx->get_param_store(), + tablet_ids, + partition_ids, + first_level_part_ids, + dtc_params))) { LOG_WARN("calculate dynamic partitions failed", K(ret)); - } else { - LOG_TRACE("dynamic partitions", K(tablet_ids), K(partition_ids), K(first_level_part_ids)); + } else if (das_location.is_dynamic_replica_select_table()) { + if (tablet_ids.count() != 1) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dynamic selected table should have only one tablet", K(tablet_ids), K(ret)); + } else { + tsc_rtdef_.dynamic_selected_tablet_id_ = tablet_ids.at(0); + } } + LOG_TRACE("dynamic calculate partitions", K(das_location.is_dynamic_replica_select_table()), + K(tablet_ids), K(partition_ids), K(first_level_part_ids), K(ret), K(das_location)); for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { ObDASTabletLoc *tablet_loc = nullptr; if (OB_FAIL(DAS_CTX(ctx_).extended_tablet_loc(*tsc_rtdef_.scan_rtdef_.table_loc_, diff --git a/src/sql/engine/table/ob_table_scan_op.h b/src/sql/engine/table/ob_table_scan_op.h index 3e16caf1b1..9dbf65fe4a 100644 --- a/src/sql/engine/table/ob_table_scan_op.h +++ b/src/sql/engine/table/ob_table_scan_op.h @@ -248,7 +248,8 @@ struct ObTableScanRtDef range_buffer_idx_(0), group_size_(0), max_group_size_(0), - attach_rtinfo_(nullptr) + attach_rtinfo_(nullptr), + dynamic_selected_tablet_id_() { } void prepare_multi_part_limit_param(); @@ -269,6 +270,13 @@ struct ObTableScanRtDef int64_t group_size_; int64_t max_group_size_; ObDASAttachRtInfo *attach_rtinfo_; + // dynamic partition pruning is used for two cases: + // 1. dynamic parameter as partitioned key + // In this case, we need to calculate tablet ids every time we get new parameters + // 2. prefer to select local replica + // In this case, tablet id only needs to be calculated once. dynamic_selected_tablet_id_ + // is used to store it to avoid duplicate calculations + ObTabletID dynamic_selected_tablet_id_; }; // table scan operator input diff --git a/src/sql/executor/ob_remote_executor_processor.cpp b/src/sql/executor/ob_remote_executor_processor.cpp index 32af639ca3..c09199bb8a 100644 --- a/src/sql/executor/ob_remote_executor_processor.cpp +++ b/src/sql/executor/ob_remote_executor_processor.cpp @@ -31,6 +31,7 @@ #include "share/scheduler/ob_tenant_dag_scheduler.h" #include "storage/tx/ob_trans_service.h" #include "sql/engine/expr/ob_expr_last_refresh_scn.h" +#include "src/rootserver/mview/ob_mview_maintenance_service.h" namespace oceanbase { @@ -432,10 +433,12 @@ int ObRemoteBaseExecuteP::execute_remote_plan(ObExecContext &exec_ctx, ObPhysicalPlanCtx *plan_ctx = exec_ctx.get_physical_plan_ctx(); ObOperator *se_op = nullptr; // static engine operator exec_ctx.set_use_temp_expr_ctx_cache(true); + rootserver::ObMViewMaintenanceService *mview_maintenance_service = + MTL(rootserver::ObMViewMaintenanceService*); FLTSpanGuard(remote_execute); - if (OB_ISNULL(plan_ctx) || OB_ISNULL(session)) { + if (OB_ISNULL(plan_ctx) || OB_ISNULL(session) || OB_ISNULL(mview_maintenance_service)) { ret = OB_ERR_UNEXPECTED; - LOG_ERROR("op is NULL", K(ret), K(plan_ctx), K(session)); + LOG_ERROR("op is NULL", K(ret), K(plan_ctx), K(session), KP(mview_maintenance_service)); } if (OB_SUCC(ret)) { int64_t retry_times = 0; @@ -468,13 +471,12 @@ int ObRemoteBaseExecuteP::execute_remote_plan(ObExecContext &exec_ctx, LOG_WARN("created operator is NULL", K(ret)); } else if (OB_FAIL(plan_ctx->reserve_param_space(plan.get_param_count()))) { LOG_WARN("reserve rescan param space failed", K(ret), K(plan.get_param_count())); - } else if (!plan.get_mview_ids().empty() && plan_ctx->get_mview_ids().empty() - && OB_FAIL(ObExprLastRefreshScn::set_last_refresh_scns(plan.get_mview_ids(), - exec_ctx.get_sql_proxy(), - exec_ctx.get_my_session(), - exec_ctx.get_das_ctx().get_snapshot().core_.version_, - plan_ctx->get_mview_ids(), - plan_ctx->get_last_refresh_scns()))) { + } else if (!plan.get_mview_ids().empty() && plan_ctx->get_mview_ids().empty() && + OB_FAIL((mview_maintenance_service->get_mview_refresh_info(plan.get_mview_ids(), + exec_ctx.get_sql_proxy(), + exec_ctx.get_das_ctx().get_snapshot().core_.version_, + plan_ctx->get_mview_ids(), + plan_ctx->get_last_refresh_scns())))) { LOG_WARN("fail to set last_refresh_scns", K(ret), K(plan.get_mview_ids())); } else { if (OB_FAIL(se_op->open())) { diff --git a/src/sql/ob_result_set.cpp b/src/sql/ob_result_set.cpp index dd9288cac7..9a58df845f 100644 --- a/src/sql/ob_result_set.cpp +++ b/src/sql/ob_result_set.cpp @@ -53,6 +53,7 @@ #include "sql/engine/dml/ob_link_op.h" #include #include "sql/engine/expr/ob_expr_last_refresh_scn.h" +#include "src/rootserver/mview/ob_mview_maintenance_service.h" using namespace oceanbase::sql; using namespace oceanbase::common; @@ -575,6 +576,8 @@ OB_INLINE int ObResultSet::do_open_plan(ObExecContext &ctx) int ret = OB_SUCCESS; ctx.reset_op_env(); exec_result_ = &(ctx.get_task_exec_ctx().get_execute_result()); + rootserver::ObMViewMaintenanceService *mview_maintenance_service = + MTL(rootserver::ObMViewMaintenanceService*); if (stmt::T_PREPARE != stmt_type_) { if (OB_FAIL(ctx.init_phy_op(physical_plan_->get_phy_operator_size()))) { LOG_WARN("fail init exec phy op ctx", K(ret)); @@ -591,15 +594,17 @@ OB_INLINE int ObResultSet::do_open_plan(ObExecContext &ctx) if (OB_FAIL(ret)) { + } else if (OB_ISNULL(mview_maintenance_service)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mview_maintenance_service is null", K(ret), KP(mview_maintenance_service)); } else if (OB_FAIL(start_stmt())) { LOG_WARN("fail start stmt", K(ret)); } else if (!physical_plan_->get_mview_ids().empty() && OB_PHY_PLAN_REMOTE != physical_plan_->get_plan_type() - && OB_FAIL(ObExprLastRefreshScn::set_last_refresh_scns(physical_plan_->get_mview_ids(), - ctx.get_sql_proxy(), - ctx.get_my_session(), - ctx.get_das_ctx().get_snapshot().core_.version_, - ctx.get_physical_plan_ctx()->get_mview_ids(), - ctx.get_physical_plan_ctx()->get_last_refresh_scns()))) { + && OB_FAIL((mview_maintenance_service->get_mview_refresh_info(physical_plan_->get_mview_ids(), + ctx.get_sql_proxy(), + ctx.get_das_ctx().get_snapshot().core_.version_, + ctx.get_physical_plan_ctx()->get_mview_ids(), + ctx.get_physical_plan_ctx()->get_last_refresh_scns())))) { LOG_WARN("fail to set last_refresh_scns", K(ret), K(physical_plan_->get_mview_ids())); } else { // for insert /*+ append */ into select clause diff --git a/src/sql/optimizer/ob_join_order.cpp b/src/sql/optimizer/ob_join_order.cpp index 43fb2ad4af..c49d8fec79 100644 --- a/src/sql/optimizer/ob_join_order.cpp +++ b/src/sql/optimizer/ob_join_order.cpp @@ -1697,7 +1697,7 @@ int ObJoinOrder::will_use_das(const uint64_t table_id, bool force_use_nlj = false; force_use_nlj = (OB_SUCCESS != (OB_E(EventTable::EN_GENERATE_PLAN_WITH_NLJ) OB_SUCCESS)); create_das_path = true; - create_basic_path = force_use_nlj ? false : true; + create_basic_path = (force_use_nlj || table_meta_info_.is_broadcast_table_) ? false : true; } else if (index_info_entry->is_index_global() && ObGlobalHint::UNSET_PARALLEL == explicit_dop) { // for global index use auto dop, create das path and basic path, after get auto dop result, prune unnecessary path create_das_path = true; @@ -13326,6 +13326,7 @@ int ObJoinOrder::compute_table_meta_info(const uint64_t table_id, table_meta_info_.part_count_ = table_partition_info_->get_phy_tbl_location_info().get_phy_part_loc_info_list().count(); table_meta_info_.schema_version_ = table_schema->get_schema_version(); + table_meta_info_.is_broadcast_table_ = table_schema->is_broadcast_table(); LOG_TRACE("after compute table meta info", K(table_meta_info_)); } if (OB_SUCC(ret)) { diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index 267fe8a06c..83293f32d6 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -10818,6 +10818,8 @@ int ObLogPlan::do_post_plan_processing() LOG_WARN("build location related tablet ids failed", K(ret)); } else if (OB_FAIL(check_das_need_keep_ordering(root))) { LOG_WARN("failed to check das need keep ordering", K(ret)); + } else if (OB_FAIL(set_major_refresh_mview_dep_table_scan(root))) { + LOG_WARN("failed to set major refresh mview dep table scan", K(ret)); } else { /*do nothing*/ } return ret; } @@ -11185,14 +11187,20 @@ int ObLogPlan::gen_das_table_location_info(ObLogTableScan *table_scan, ObSEArray all_filters; bool has_dppr = false; ObOptimizerContext *opt_ctx = &get_optimizer_context(); + const ObCostTableScanInfo *est_cost_info = NULL; + const ObTableMetaInfo *table_meta_info = NULL; if (OB_ISNULL(table_scan) || OB_ISNULL(table_partition_info) || + OB_ISNULL(opt_ctx) || OB_ISNULL(sql_schema_guard = opt_ctx->get_sql_schema_guard()) || OB_ISNULL(stmt = table_scan->get_stmt()) || OB_ISNULL(table_item = stmt->get_table_item_by_id(table_scan->get_table_id())) || - OB_ISNULL(table_scan->get_strong_sharding())) { + OB_ISNULL(table_scan->get_strong_sharding()) || + OB_ISNULL(est_cost_info = table_scan->get_est_cost_info()) || + OB_ISNULL(table_meta_info = est_cost_info->table_meta_info_)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("get unexpected null", K(sql_schema_guard), K(stmt), K(ret)); + LOG_WARN("get unexpected null", K(sql_schema_guard), K(stmt), K(opt_ctx), + K(est_cost_info), K(table_meta_info), K(ret)); } else if (!table_scan->use_das() || !table_scan->is_match_all()) { // do nothing } else if (OB_FAIL(append_array_no_dup(all_filters, table_scan->get_range_conditions()))) { @@ -11203,14 +11211,14 @@ int ObLogPlan::gen_das_table_location_info(ObLogTableScan *table_scan, has_dppr))) { LOG_WARN("failed to find das dppr filter exprs", K(ret)); } else if (!has_dppr) { - // do nothing + // do nothing } else { SMART_VAR(ObTableLocation, das_location) { const ObDataTypeCastParams dtc_params = ObBasicSessionInfo::create_dtc_params(opt_ctx->get_session_info()); int64_t ref_table_id = table_scan->get_is_index_global() ? - table_scan->get_index_table_id() : - table_scan->get_ref_table_id(); + table_scan->get_index_table_id() : + table_scan->get_ref_table_id(); if (OB_FAIL(das_location.init(*sql_schema_guard, *stmt, opt_ctx->get_exec_ctx(), @@ -11221,11 +11229,13 @@ int ObLogPlan::gen_das_table_location_info(ObLogTableScan *table_scan, dtc_params, false))) { LOG_WARN("fail to init table location", K(ret), K(all_filters)); - } else if (das_location.is_all_partition()) { + } else if (OB_FALSE_IT(das_location.set_use_das(true))) { + } else if (OB_FALSE_IT(das_location.set_broadcast_table(table_meta_info->is_broadcast_table_))) { + } else if (das_location.is_all_partition() && + !das_location.is_dynamic_replica_select_table()) { // do nothing } else { - das_location.set_has_dynamic_exec_param(true); - das_location.set_use_das(true); + das_location.set_has_dynamic_exec_param(has_dppr); table_partition_info->set_table_location(das_location); } } @@ -11569,6 +11579,97 @@ int ObLogPlan::check_das_need_keep_ordering(ObLogicalOperator *op) return ret; } +int ObLogPlan::set_major_refresh_mview_dep_table_scan(ObLogicalOperator *op) +{ + int ret = OB_SUCCESS; + bool for_fast_refresh = false; + ObSQLSessionInfo *session = get_optimizer_context().get_session_info(); + if (OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(session)); + } else if (OB_FALSE_IT(for_fast_refresh = session->get_ddl_info().is_major_refreshing_mview())) { + } else if (OB_FAIL(set_major_refresh_mview_dep_table_scan(for_fast_refresh, false, op))) { + LOG_WARN("failed to set major refresh mview dep table scan", K(ret)); + } + return ret; +} + +int ObLogPlan::set_major_refresh_mview_dep_table_scan(bool for_fast_refresh, + bool for_rt_mview, + ObLogicalOperator *op) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(op)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null param", K(ret)); + } else if (log_op_def::LOG_SET == op->get_type() && !for_rt_mview) { + if (OB_FAIL(is_major_refresh_rt_mview(op->get_stmt(), + get_optimizer_context().get_sql_schema_guard(), + for_rt_mview))) { + LOG_WARN("failed to check is major refresh rt mview", K(ret)); + } + } else if (log_op_def::LOG_TABLE_SCAN == op->get_type() && (for_fast_refresh || for_rt_mview)) { + ObLogTableScan *scan = static_cast(op); + const TableItem *table_item = NULL; + if (OB_ISNULL(op->get_stmt()) || OB_ISNULL(table_item = op->get_stmt()->get_table_item_by_id(scan->get_table_id()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null param", K(ret)); + } else if (is_virtual_table(scan->get_ref_table_id()) + || is_inner_table(scan->get_ref_table_id()) + || MATERIALIZED_VIEW == table_item->table_type_) { + /* do nothing */ + } else if (for_fast_refresh) { + scan->set_for_mr_mv_refresh(); + LOG_TRACE("set as major refresh mview dep table scan for refresh", K(scan->get_table_name())); + } else { + scan->set_for_mr_rt_mv(); + LOG_TRACE("set as major refresh mview dep table scan for rt-mview", K(scan->get_table_name())); + } + } + for (int i = 0; OB_SUCC(ret) && i < op->get_num_of_child(); ++i) { + if (OB_FAIL(SMART_CALL(set_major_refresh_mview_dep_table_scan(for_fast_refresh, + for_rt_mview, + op->get_child(i))))) { + LOG_WARN("failed to set major refresh mview dep table scan", K(ret)); + } + } + return ret; +} + +int ObLogPlan::is_major_refresh_rt_mview(const ObDMLStmt *stmt, + const ObSqlSchemaGuard *sql_schema_guard, + bool &is_mr_rt_mview) +{ + int ret = OB_SUCCESS; + is_mr_rt_mview = false; + const ObSelectStmt *sel_stmt = dynamic_cast(stmt); + if (NULL == sel_stmt || !sel_stmt->is_expanded_mview()) { + /* do nothing */ + } else if (OB_ISNULL(sel_stmt = sel_stmt->get_set_query(0)) || OB_ISNULL(sql_schema_guard)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null param", K(ret), K(sel_stmt), K(sql_schema_guard)); + } else { + const ObIArray &table_items = sel_stmt->get_table_items(); + const TableItem *table_item = NULL; + const ObTableSchema *mview_schema = NULL; + for (int i = 0; NULL == table_item && i < table_items.count(); ++i) { + if (OB_NOT_NULL(table_items.at(i)) && MATERIALIZED_VIEW == table_items.at(i)->table_type_) { + table_item = table_items.at(i); + } + } + if (OB_FAIL(ret) || NULL == table_item) { + } else if (OB_FAIL(sql_schema_guard->get_table_schema(table_item->mview_id_, mview_schema))) { + LOG_WARN("failed to get table schema", K(ret)); + } else if (OB_ISNULL(mview_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null mview schema", K(ret)); + } else { + is_mr_rt_mview = mview_schema->mv_major_refresh(); + } + } + return ret; +} + int ObLogPlan::check_das_need_scan_with_domain_id(ObLogicalOperator *op) { int ret = OB_SUCCESS; diff --git a/src/sql/optimizer/ob_log_plan.h b/src/sql/optimizer/ob_log_plan.h index 8a044a0d39..78980a4250 100644 --- a/src/sql/optimizer/ob_log_plan.h +++ b/src/sql/optimizer/ob_log_plan.h @@ -266,6 +266,14 @@ public: int check_das_need_keep_ordering(ObLogicalOperator *op); int check_das_need_scan_with_domain_id(ObLogicalOperator *op); + int set_major_refresh_mview_dep_table_scan(ObLogicalOperator *op); + int set_major_refresh_mview_dep_table_scan(bool for_fast_refresh, + bool for_rt_mview, + ObLogicalOperator *op); + int is_major_refresh_rt_mview(const ObDMLStmt *set_stmt, + const ObSqlSchemaGuard *sql_schema_guard, + bool &is_mr_rt_mview); + int gen_das_table_location_info(ObLogTableScan *table_scan, ObTablePartitionInfo *&table_partition_info); diff --git a/src/sql/optimizer/ob_log_table_scan.cpp b/src/sql/optimizer/ob_log_table_scan.cpp index 1125229201..c801074300 100644 --- a/src/sql/optimizer/ob_log_table_scan.cpp +++ b/src/sql/optimizer/ob_log_table_scan.cpp @@ -548,19 +548,19 @@ int ObLogTableScan::generate_access_exprs() if (OB_ISNULL(expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret)); - } else if ((T_ORA_ROWSCN == expr->get_expr_type()) - && static_cast(expr)->get_table_id() == table_id_) { - if (OB_FAIL(access_exprs_.push_back(expr))) { - LOG_WARN("fail to push back expr", K(ret)); - } - } else if ((T_PSEUDO_EXTERNAL_FILE_URL == expr->get_expr_type()) - && static_cast(expr)->get_table_id() == table_id_) { + } else if (static_cast(expr)->get_table_id() != table_id_) { + /* do nothing */ + } else if (T_ORA_ROWSCN != expr->get_expr_type() + && T_PSEUDO_EXTERNAL_FILE_URL != expr->get_expr_type() + && T_PSEUDO_OLD_NEW_COL != expr->get_expr_type()) { + /* do nothing */ + } else if (OB_FAIL(access_exprs_.push_back(expr))) { + LOG_WARN("fail to push back expr", K(ret)); + } else if (T_PSEUDO_EXTERNAL_FILE_URL == expr->get_expr_type()) { if (OB_FAIL(add_var_to_array_no_dup(ext_file_column_exprs_, expr))) { LOG_WARN("fail to push back expr", K(ret)); } else if (OB_FAIL(add_var_to_array_no_dup(output_exprs_, expr))) { LOG_WARN("fail to push back expr", K(ret)); - } else if (OB_FAIL(access_exprs_.push_back(expr))) { //add access expr temp - LOG_WARN("fail to push back expr", K(ret)); } } } @@ -1176,7 +1176,8 @@ int ObLogTableScan::index_back_check() column_found = false; } else if (ob_is_geometry_tc(expr->get_data_type())) { // 在此处先标记为需要index_back，具体是否需要需要结合谓词来判断。 column_found = false; - } else if (T_PSEUDO_GROUP_ID == expr->get_expr_type()) { + } else if (T_PSEUDO_GROUP_ID == expr->get_expr_type() || + T_PSEUDO_OLD_NEW_COL == expr->get_expr_type()) { // do nothing } else if (OB_UNLIKELY(!expr->is_column_ref_expr())) { ret = OB_ERR_UNEXPECTED; diff --git a/src/sql/optimizer/ob_log_table_scan.h b/src/sql/optimizer/ob_log_table_scan.h index 449c4457aa..e5384eb211 100644 --- a/src/sql/optimizer/ob_log_table_scan.h +++ b/src/sql/optimizer/ob_log_table_scan.h @@ -224,7 +224,8 @@ public: multivalue_col_idx_(common::OB_INVALID_ID), multivalue_type_(-1), is_tsc_with_vid_(false), - rowkey_vid_tid_(common::OB_INVALID_ID) + rowkey_vid_tid_(common::OB_INVALID_ID), + mr_mv_scan_(common::ObQueryFlag::NormalMode) { } @@ -687,6 +688,9 @@ public: const ObColumnRefRawExpr *col_expr, PushdownFilterMonotonicity &mono, ObIArray &assist_exprs) const; + void set_for_mr_mv_refresh() { mr_mv_scan_ = common::ObQueryFlag::RefreshMode; } + void set_for_mr_rt_mv() { mr_mv_scan_ = common::ObQueryFlag::RealTimeMode; } + common::ObQueryFlag::MRMVScanMode get_mr_mv_scan() const { return mr_mv_scan_; } bool use_index_merge() const; const ObIArray &get_full_filters() const { return full_filters_; } @@ -887,6 +891,8 @@ protected: // memeber variables uint64_t rowkey_vid_tid_; // end for table scan with vid + common::ObQueryFlag::MRMVScanMode mr_mv_scan_; // used for major refresh mview fast refresh and real-time mview + // disallow copy and assign DISALLOW_COPY_AND_ASSIGN(ObLogTableScan); }; diff --git a/src/sql/optimizer/ob_opt_est_cost_model.h b/src/sql/optimizer/ob_opt_est_cost_model.h index 457544c382..90f87f2956 100644 --- a/src/sql/optimizer/ob_opt_est_cost_model.h +++ b/src/sql/optimizer/ob_opt_est_cost_model.h @@ -61,7 +61,8 @@ struct ObTableMetaInfo row_count_(0), has_opt_stat_(false), micro_block_count_(-1), - table_type_(share::schema::MAX_TABLE_TYPE) + table_type_(share::schema::MAX_TABLE_TYPE), + is_broadcast_table_(false) { } virtual ~ObTableMetaInfo() { } @@ -90,6 +91,7 @@ struct ObTableMetaInfo bool has_opt_stat_; int64_t micro_block_count_; // main table micro block count share::schema::ObTableType table_type_; + bool is_broadcast_table_; private: DISALLOW_COPY_AND_ASSIGN(ObTableMetaInfo); }; diff --git a/src/sql/optimizer/ob_table_location.cpp b/src/sql/optimizer/ob_table_location.cpp index 260207f703..e7d20bc3f7 100644 --- a/src/sql/optimizer/ob_table_location.cpp +++ b/src/sql/optimizer/ob_table_location.cpp @@ -726,6 +726,7 @@ int ObTableLocation::assign(const ObTableLocation &other) tablet_id_ = other.tablet_id_; object_id_ = other.object_id_; check_no_partition_ = other.check_no_partition_; + is_broadcast_table_ = other.is_broadcast_table_; if (OB_FAIL(loc_meta_.assign(other.loc_meta_))) { LOG_WARN("assign loc meta failed", K(ret), K(other.loc_meta_)); } @@ -856,6 +857,7 @@ void ObTableLocation::reset() tablet_id_.reset(); object_id_ = OB_INVALID_ID; check_no_partition_ = false; + is_broadcast_table_ = false; } int ObTableLocation::init(share::schema::ObSchemaGetterGuard &schema_guard, const ObDMLStmt &stmt, @@ -1018,7 +1020,7 @@ int ObTableLocation::init_table_location(ObExecContext &exec_ctx, exec_ctx.get_sql_ctx(), is_weak_read))) { LOG_WARN("get is weak read failed", K(ret)); - } else if (ObDuplicateScope::DUPLICATE_SCOPE_NONE != table_schema->get_duplicate_scope()) { + } else if (table_schema->is_duplicate_table()) { loc_meta_.is_dup_table_ = 1; } if (OB_SUCC(ret)) { @@ -1323,7 +1325,7 @@ int ObTableLocation::init( bool is_weak_read = false; if (OB_FAIL(get_is_weak_read(stmt, session_info, exec_ctx->get_sql_ctx(), is_weak_read))) { LOG_WARN("get is weak read failed", K(ret)); - } else if (ObDuplicateScope::DUPLICATE_SCOPE_NONE != table_schema->get_duplicate_scope()) { + } else if (table_schema->is_duplicate_table()) { loc_meta_.is_dup_table_ = 1; } if (is_dml_table) { @@ -4860,6 +4862,7 @@ OB_DEF_SERIALIZE(ObTableLocation) OB_UNIS_ENCODE(related_list_); OB_UNIS_ENCODE(table_type_); OB_UNIS_ENCODE(check_no_partition_); + OB_UNIS_ENCODE(is_broadcast_table_); return ret; } @@ -4938,6 +4941,7 @@ OB_DEF_SERIALIZE_SIZE(ObTableLocation) OB_UNIS_ADD_LEN(related_list_); OB_UNIS_ADD_LEN(table_type_); OB_UNIS_ADD_LEN(check_no_partition_); + OB_UNIS_ADD_LEN(is_broadcast_table_); return len; } @@ -5094,6 +5098,7 @@ OB_DEF_DESERIALIZE(ObTableLocation) OB_UNIS_DECODE(related_list_); OB_UNIS_DECODE(table_type_); OB_UNIS_DECODE(check_no_partition_); + OB_UNIS_DECODE(is_broadcast_table_); return ret; } diff --git a/src/sql/optimizer/ob_table_location.h b/src/sql/optimizer/ob_table_location.h index 49d235ad9e..b6c8305167 100644 --- a/src/sql/optimizer/ob_table_location.h +++ b/src/sql/optimizer/ob_table_location.h @@ -498,7 +498,8 @@ public: tablet_id_(ObTabletID::INVALID_TABLET_ID), object_id_(OB_INVALID_ID), related_list_(allocator_), - check_no_partition_(false) + check_no_partition_(false), + is_broadcast_table_(false) { } @@ -547,7 +548,8 @@ public: tablet_id_(ObTabletID::INVALID_TABLET_ID), object_id_(OB_INVALID_ID), related_list_(allocator_), - check_no_partition_(false) + check_no_partition_(false), + is_broadcast_table_(false) { } virtual ~ObTableLocation() { reset(); } @@ -722,6 +724,12 @@ public: uint64_t ref_table_id, ObDASTableLoc *&table_loc); bool is_duplicate_table() const { return loc_meta_.is_dup_table_; } + bool is_dynamic_replica_select_table() const { return (is_duplicate_table() || get_is_broadcast_table()) && + !is_partitioned(); } + void set_broadcast_table(const bool is_broadcast_table) { + is_broadcast_table_ = is_broadcast_table; + } + bool get_is_broadcast_table() const { return is_broadcast_table_; } bool is_duplicate_table_not_in_dml() const { return loc_meta_.is_dup_table_ && !loc_meta_.select_leader_; } void set_duplicate_type(ObDuplicateType v) { duplicate_type_to_loc_meta(v, loc_meta_); } @@ -1192,6 +1200,7 @@ private: ObObjectID object_id_; common::ObList related_list_; bool check_no_partition_; + bool is_broadcast_table_; }; } diff --git a/src/sql/optimizer/ob_table_partition_info.cpp b/src/sql/optimizer/ob_table_partition_info.cpp index 410e605f54..dc1c139005 100644 --- a/src/sql/optimizer/ob_table_partition_info.cpp +++ b/src/sql/optimizer/ob_table_partition_info.cpp @@ -66,7 +66,7 @@ int ObTablePartitionInfo::init_table_location(ObSqlSchemaGuard &schema_guard, const ObTableSchema *table_schema = NULL; if (OB_FAIL(schema_guard.get_table_schema(table_id, ref_table_id, &stmt, table_schema))) { LOG_WARN("fail to get table schema", K(ref_table_id), K(ret)); - } else if (ObDuplicateScope::DUPLICATE_SCOPE_NONE != table_schema->get_duplicate_scope()) { + } else if (table_schema->is_duplicate_table()) { //如果复制表本身有改动, 只能选择leader, 不再设置duplicate table属性 candi_table_loc_.set_duplicate_type(is_dml_table ? ObDuplicateType::DUPLICATE_IN_DML : ObDuplicateType::DUPLICATE); diff --git a/src/sql/parser/non_reserved_keywords_mysql_mode.c b/src/sql/parser/non_reserved_keywords_mysql_mode.c index e91179be0f..a63c9ad6ea 100644 --- a/src/sql/parser/non_reserved_keywords_mysql_mode.c +++ b/src/sql/parser/non_reserved_keywords_mysql_mode.c @@ -741,6 +741,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"replica_num", REPLICA_NUM}, {"replica_type", REPLICA_TYPE}, {"duplicate_scope", DUPLICATE_SCOPE}, + {"duplicate_read_consistency", DUPLICATE_READ_CONSISTENCY}, {"replication", REPLICATION}, {"report", REPORT}, {"require", REQUIRE}, diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 8a5f602dd9..29b73e2495 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -281,7 +281,7 @@ END_P SET_VAR DELIMITER DAG DATA DATAFILE DATA_DISK_SIZE DATA_SOURCE DATA_TABLE_ID DATE DATE_ADD DATE_SUB DATETIME DAY DEALLOCATE DECRYPTION DEFAULT_AUTH DEFAULT_LOB_INROW_THRESHOLD DEFINER DELAY DELAY_KEY_WRITE DEPTH DES_KEY_FILE DENSE_RANK DESCRIPTION DESTINATION DIAGNOSTICS - DIRECTORY DISABLE DISALLOW DISCARD DISK DISKGROUP DO DOT DUMP DUMPFILE DUPLICATE DUPLICATE_SCOPE DYNAMIC + DIRECTORY DISABLE DISALLOW DISCARD DISK DISKGROUP DO DOT DUMP DUMPFILE DUPLICATE DUPLICATE_SCOPE DUPLICATE_READ_CONSISTENCY DYNAMIC DATABASE_ID DEFAULT_TABLEGROUP DISCONNECT DEMAND EFFECTIVE EMPTY ENABLE ENABLE_ARBITRATION_SERVICE ENABLE_EXTENDED_ROWID ENCRYPTED ENCRYPTION END ENDPOINT ENDS ENFORCED ENGINE_ ENGINES ENUM ENTITY ERROR_CODE ERROR_P ERRORS ESTIMATE @@ -7515,6 +7515,11 @@ TABLE_MODE opt_equal_mark STRING_VALUE int_node->value_ = 2; malloc_non_terminal_node($$, result->malloc_pool_, T_EXTERNAL_TABLE_AUTO_REFRESH, 1, int_node); } +| DUPLICATE_READ_CONSISTENCY opt_equal_mark STRING_VALUE +{ + (void)($2); + malloc_non_terminal_node($$, result->malloc_pool_, T_DUPLICATE_READ_CONSISTENCY, 1, $3); +} | MAX_ROWS opt_equal_mark INTNUM { (void)($2); @@ -23748,6 +23753,7 @@ ACCESS_INFO | DUMPFILE | DUPLICATE | DUPLICATE_SCOPE +| DUPLICATE_READ_CONSISTENCY | DYNAMIC | DEFAULT_TABLEGROUP | DEFAULT_LOB_INROW_THRESHOLD diff --git a/src/sql/printer/ob_raw_expr_printer.cpp b/src/sql/printer/ob_raw_expr_printer.cpp index bf3dd2e606..1d1db638d3 100644 --- a/src/sql/printer/ob_raw_expr_printer.cpp +++ b/src/sql/printer/ob_raw_expr_printer.cpp @@ -4079,6 +4079,8 @@ int ObRawExprPrinter::print(ObPseudoColumnRawExpr *expr) SET_SYMBOL_IF_EMPTY("connect_by_iscycle"); case T_ORA_ROWSCN : SET_SYMBOL_IF_EMPTY("ora_rowscn"); + case T_PSEUDO_OLD_NEW_COL : + SET_SYMBOL_IF_EMPTY("old_new$$"); case T_CONNECT_BY_ISLEAF : { SET_SYMBOL_IF_EMPTY("connect_by_isleaf"); if (0 != expr->get_param_count()) { diff --git a/src/sql/resolver/ddl/ob_alter_table_resolver.cpp b/src/sql/resolver/ddl/ob_alter_table_resolver.cpp index 033ce0e499..b3fa162f71 100644 --- a/src/sql/resolver/ddl/ob_alter_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_alter_table_resolver.cpp @@ -242,7 +242,7 @@ int ObAlterTableResolver::resolve(const ParseNode &parse_tree) LOG_USER_ERROR(OB_OP_NOT_ALLOW, "alter table localiy and tablegroup at the same time"); } else if (OB_FAIL(set_table_options())) { SQL_RESV_LOG(WARN, "failed to set table options.", K(ret)); - } else if ((table_schema_->has_mlog_table() || table_schema_->is_mlog_table()) + } else if ((table_schema_->required_by_mview_refresh() || table_schema_->is_mlog_table()) && OB_FAIL(ObResolverUtils::check_allowed_alter_operations_for_mlog( alter_table_stmt->get_tenant_id(), alter_table_stmt->get_alter_table_arg(), @@ -406,7 +406,7 @@ int ObAlterTableResolver::set_table_options() alter_table_schema.set_read_only(read_only_); alter_table_schema.set_row_store_type(row_store_type_); alter_table_schema.set_store_format(store_format_); - alter_table_schema.set_duplicate_scope(duplicate_scope_); + alter_table_schema.set_duplicate_attribute(duplicate_scope_, duplicate_read_consistency_); alter_table_schema.set_enable_row_movement(enable_row_movement_); alter_table_schema.set_storage_format_version(storage_format_version_); alter_table_schema.set_table_mode_struct(table_mode_); diff --git a/src/sql/resolver/ddl/ob_create_index_resolver.cpp b/src/sql/resolver/ddl/ob_create_index_resolver.cpp index 25adf7d771..7886deba6d 100644 --- a/src/sql/resolver/ddl/ob_create_index_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_index_resolver.cpp @@ -614,6 +614,11 @@ int ObCreateIndexResolver::resolve(const ParseNode &parse_tree) if (OB_TABLE_NOT_EXIST == ret) { ret = OB_ERR_UNEXPECTED; // rewrite errno } + } else if (mv_container_table_schema->mv_major_refresh()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("create index on major refresh materialized view is not supported", KR(ret), + K(tbl_schema->get_table_name())); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "create index on major refresh materialized view is"); } else { is_oracle_temp_table_ = (tbl_schema->is_oracle_tmp_table()); ObTableSchema &index_schema = crt_idx_stmt->get_create_index_arg().index_schema_; diff --git a/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp b/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp index 765fbaf9fa..f890c6a44a 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp +++ b/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp @@ -109,7 +109,7 @@ int ObCreateTableResolverBase::set_table_option_to_schema(ObTableSchema &table_s table_schema.set_tablegroup_id(OB_INVALID_ID); table_schema.set_table_id(table_id_); table_schema.set_read_only(read_only_); - table_schema.set_duplicate_scope(duplicate_scope_); + table_schema.set_duplicate_attribute(duplicate_scope_, duplicate_read_consistency_); table_schema.set_enable_row_movement(enable_row_movement_); table_schema.set_table_mode_struct(table_mode_); table_schema.set_encryption_str(encryption_); diff --git a/src/sql/resolver/ddl/ob_create_view_resolver.cpp b/src/sql/resolver/ddl/ob_create_view_resolver.cpp index b41fe18f2c..0b0dc08f4a 100644 --- a/src/sql/resolver/ddl/ob_create_view_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_view_resolver.cpp @@ -417,11 +417,6 @@ int ObCreateViewResolver::resolve(const ParseNode &parse_tree) mv_ainfo->container_table_schema_, csts))) { LOG_WARN("fail do resolve for materialized view", K(ret)); - } else if (OB_FAIL(resolve_mv_options(select_stmt, - parse_tree.children_[MVIEW_NODE], - mv_ainfo->mv_refresh_info_, - table_schema))) { - LOG_WARN("fail to resolve mv options", K(ret)); } else if (OB_FAIL(load_mview_dep_session_vars(*session_info_, select_stmt, table_schema.get_local_session_var()))) { LOG_WARN("fail to load mview dep session variables", K(ret)); } else if (!tenant_config.is_valid()) { @@ -431,6 +426,12 @@ int ObCreateViewResolver::resolve(const ParseNode &parse_tree) && OB_FAIL(resolve_column_group_helper(parse_tree.children_[COLUMN_GROUP_NODE], mv_ainfo->container_table_schema_))) { LOG_WARN("fail to resolve column group", KR(ret)); + } else if (OB_FAIL(resolve_mv_options(select_stmt, + parse_tree.children_[MVIEW_NODE], + mv_ainfo->mv_refresh_info_, + table_schema, + mv_ainfo->container_table_schema_))) { + LOG_WARN("fail to resolve mv options", K(ret)); } if (OB_SUCC(ret)) { if (OB_FAIL(resolve_hints(parse_tree.children_[HINT_NODE], *stmt, mv_ainfo->container_table_schema_))) { @@ -1296,7 +1297,8 @@ int ObCreateViewResolver::create_alias_names_auto( int ObCreateViewResolver::resolve_mv_options(const ObSelectStmt *stmt, ParseNode *options_node, ObMVRefreshInfo &refresh_info, - ObTableSchema &table_schema) + ObTableSchema &table_schema, + ObTableSchema &container_table_schema) { int ret = OB_SUCCESS; refresh_info.refresh_method_ = ObMVRefreshMethod::FORCE; //default method is force @@ -1318,20 +1320,34 @@ int ObCreateViewResolver::resolve_mv_options(const ObSelectStmt *stmt, table_schema.set_mv_enable_query_rewrite(ObMVEnableQueryRewriteFlag::IS_MV_ENABLE_QUERY_REWRITE); } } - if (OB_FAIL(ret)) { - } else if ((table_schema.mv_on_query_computation() || ObMVRefreshMethod::FAST == refresh_info.refresh_method_) - && OB_FAIL(ObMVChecker::check_mv_fast_refresh_valid(stmt, params_.stmt_factory_, - params_.expr_factory_, - params_.session_info_))) { - // When creating an MV, which can not be fast refreshed, with both fast refresh - // and on query computation, we should return CAN_NOT_ON_QUERY_COMPUTE - if (table_schema.mv_on_query_computation() && OB_ERR_MVIEW_CAN_NOT_FAST_REFRESH == ret) { - ret = OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE; + if (OB_SUCC(ret)) { + if ((table_schema.mv_on_query_computation() || + ObMVRefreshMethod::FAST == refresh_info.refresh_method_)) { + ObMVRefreshableType refresh_type = OB_MV_REFRESH_INVALID; + if (OB_FAIL(ObMVChecker::check_mv_fast_refresh_type( + stmt, params_.stmt_factory_, params_.expr_factory_, params_.session_info_, + container_table_schema, refresh_type))) { + LOG_WARN("fail to check mv type", KR(ret)); + } else if (OB_UNLIKELY(!IS_VALID_FAST_REFRESH_TYPE(refresh_type))) { + // When creating an MV, which can not be fast refreshed, with both fast refresh + // and on query computation, we should return CAN_NOT_ON_QUERY_COMPUTE + if (table_schema.mv_on_query_computation()) { + ret = OB_ERR_MVIEW_CAN_NOT_ON_QUERY_COMPUTE; + } else { + ret = OB_ERR_MVIEW_CAN_NOT_FAST_REFRESH; + } + LOG_WARN("fast refresh is not supported for this mv", KR(ret), K(refresh_type)); + } else if (OB_MV_FAST_REFRESH_MAJOR_REFRESH_MJV == refresh_type) { + table_schema.set_mv_major_refresh(IS_MV_MAJOR_REFRESH); + container_table_schema.set_mv_major_refresh(IS_MV_MAJOR_REFRESH); + refresh_info.refresh_mode_ = ObMVRefreshMode::MAJOR_COMPACTION; + LOG_INFO("[MAJ_REF_MV] match major refresh mv", K(table_schema.get_table_name())); + } + if (OB_SUCC(ret) && table_schema.mv_on_query_computation() && + OB_FAIL(check_on_query_computation_supported(stmt))) { + LOG_WARN("fail to check on query computation mv column type", KR(ret)); + } } - LOG_WARN("fail to check fast refresh valid", K(ret)); - } else if (table_schema.mv_on_query_computation() - && OB_FAIL(check_on_query_computation_supported(stmt))) { - LOG_WARN("fail to check on query computation mv column type", K(ret)); } return ret; } diff --git a/src/sql/resolver/ddl/ob_create_view_resolver.h b/src/sql/resolver/ddl/ob_create_view_resolver.h index 927b8d1fde..11e8f4a5d4 100644 --- a/src/sql/resolver/ddl/ob_create_view_resolver.h +++ b/src/sql/resolver/ddl/ob_create_view_resolver.h @@ -100,7 +100,8 @@ private: int resolve_mv_options(const ObSelectStmt *stmt, ParseNode *options_node, ObMVRefreshInfo &refresh_info, - ObTableSchema &table_schema); + ObTableSchema &table_schema, + ObTableSchema &container_table_schema); int resolve_mv_refresh_info(ParseNode *refresh_info_node, ObMVRefreshInfo &refresh_info); diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index c26f5ce3b9..0f944ab36f 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -108,6 +108,7 @@ ObDDLResolver::ObDDLResolver(ObResolverParams ¶ms) locality_(), is_random_primary_zone_(false), duplicate_scope_(share::ObDuplicateScope::DUPLICATE_SCOPE_NONE), + duplicate_read_consistency_(share::ObDuplicateReadConsistency::STRONG), enable_row_movement_(false), encryption_(), tablespace_id_(OB_INVALID_ID), @@ -1212,6 +1213,7 @@ int ObDDLResolver::resolve_table_id_pre(ParseNode *node) int ObDDLResolver::resolve_table_options(ParseNode *node, bool is_index_option) { int ret = OB_SUCCESS; + bool exist_duplicate_read_consistency = false; if (NULL != node) { ParseNode *option_node = NULL; int32_t num = 0; @@ -1231,9 +1233,18 @@ int ObDDLResolver::resolve_table_options(ParseNode *node, bool is_index_option) SQL_RESV_LOG(WARN, "node is null", K(ret)); } else if (OB_FAIL(resolve_table_option(option_node, is_index_option))) { SQL_RESV_LOG(WARN, "resolve table option failed", K(ret)); + } else if (T_DUPLICATE_READ_CONSISTENCY == option_node->type_) { + exist_duplicate_read_consistency = true; } } } + if (OB_SUCC(ret) && exist_duplicate_read_consistency) { + if (!ObDuplicateScopeChecker::is_valid_duplicate_scope(duplicate_scope_)) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "duplicate read consistency with invalid duplicate scope"); + } + } + if (OB_SUCC(ret)) { if (CHARSET_INVALID == charset_type_ && CS_TYPE_INVALID == collation_type_ ) { @@ -2507,6 +2518,35 @@ int ObDDLResolver::resolve_table_option(const ParseNode *option_node, const bool } break; } + case T_DUPLICATE_READ_CONSISTENCY: { + ObString duplicate_read_consistency_str; + share::ObDuplicateReadConsistency read_consistency = share::ObDuplicateReadConsistency::MAX; + uint64_t tenant_data_version = 0; + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) { + LOG_WARN("get tenant data version failed", K(ret)); + } else if (tenant_data_version < DATA_VERSION_4_3_4_0) { // todo siyu use data version 434 + LOG_WARN("create table with duplicate_read_consistency option is not supported in data version less than 4.3.4", K(ret), K(tenant_data_version)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "data version is less than 4.3.4, create table with duplicate_read_consistency option"); + } else if (nullptr == option_node->children_ || 1 != option_node->num_child_) { + ret = common::OB_INVALID_ARGUMENT; + SQL_RESV_LOG(WARN, "invalid duplicate read consistency arg", K(ret), "num_child", option_node->num_child_); + } else if (nullptr == option_node->children_[0]) { + ret = OB_ERR_UNEXPECTED; + SQL_RESV_LOG(WARN, "option node child is null", K(ret)); + } else { + duplicate_read_consistency_str.assign_ptr(const_cast(option_node->children_[0]->str_value_), + static_cast(option_node->children_[0]->str_len_)); + duplicate_read_consistency_str = duplicate_read_consistency_str.trim(); + if (OB_FAIL(ObDuplicateReadConsistencyChecker::convert_duplicate_read_consistency_string( + duplicate_read_consistency_str, read_consistency))) { + SQL_RESV_LOG(WARN, "fail to convert duplicate read consistency string", K(ret)); + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "duplicate_read_consistency"); + } else { + duplicate_read_consistency_ = read_consistency; + } + } + break; + } case T_LOCALITY: { if (NULL == option_node->children_ || option_node->num_child_ != 2) { ret = common::OB_INVALID_ARGUMENT; diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.h b/src/sql/resolver/ddl/ob_ddl_resolver.h index de55ed100c..fd6ae25b9e 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.h +++ b/src/sql/resolver/ddl/ob_ddl_resolver.h @@ -1075,6 +1075,7 @@ protected: common::ObString locality_; bool is_random_primary_zone_; share::ObDuplicateScope duplicate_scope_; + share::ObDuplicateReadConsistency duplicate_read_consistency_; bool enable_row_movement_; share::schema::ObTableMode table_mode_; common::ObString encryption_; diff --git a/src/sql/resolver/ddl/ob_rename_table_resolver.cpp b/src/sql/resolver/ddl/ob_rename_table_resolver.cpp index 71e8bfa7ff..0717be80e2 100644 --- a/src/sql/resolver/ddl/ob_rename_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_rename_table_resolver.cpp @@ -137,10 +137,10 @@ int ObRenameTableResolver::resolve_rename_action(const ParseNode &rename_action_ ret = OB_NOT_SUPPORTED; LOG_WARN("rename materialized view log is not supported", KR(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "rename materialized view log is"); - } else if (table_schema->has_mlog_table()) { + } else if (table_schema->required_by_mview_refresh()) { ret = OB_NOT_SUPPORTED; - LOG_WARN("rename table with materialized view log is not supported", KR(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "rename table with materialized view log is"); + LOG_WARN("rename table required by materialized view refresh is not supported", KR(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "rename table required by materialized view refresh is"); } } } diff --git a/src/sql/resolver/ddl/ob_truncate_table_resolver.cpp b/src/sql/resolver/ddl/ob_truncate_table_resolver.cpp index 8915ae6870..dfd52b95b9 100644 --- a/src/sql/resolver/ddl/ob_truncate_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_truncate_table_resolver.cpp @@ -140,11 +140,11 @@ int ObTruncateTableResolver::resolve(const ParseNode &parser_tree) SQL_RESV_LOG(WARN, "truncate materialized view log is not supported", KR(ret), K(orig_table_schema->get_table_name())); LOG_USER_ERROR(OB_NOT_SUPPORTED, "truncate materialized view log is"); - } else if (orig_table_schema->has_mlog_table()) { + } else if (orig_table_schema->required_by_mview_refresh()) { ret = OB_NOT_SUPPORTED; - SQL_RESV_LOG(WARN, "truncate table with materialized view log is not supported", + SQL_RESV_LOG(WARN, "truncate table required by materialized view refresh is not supported", KR(ret), K(orig_table_schema->get_table_name())); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "truncate table with materialized view log is"); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "truncate table required by materialized view refresh is"); } } } diff --git a/src/sql/resolver/dml/ob_column_namespace_checker.cpp b/src/sql/resolver/dml/ob_column_namespace_checker.cpp index 5a5544365b..5f2d05e46b 100644 --- a/src/sql/resolver/dml/ob_column_namespace_checker.cpp +++ b/src/sql/resolver/dml/ob_column_namespace_checker.cpp @@ -297,8 +297,13 @@ int ObColumnNamespaceChecker::check_column_exists(const TableItem &table_item, c LOG_WARN("params_.session_info_ is null", K(ret)); } else if (table_item.is_basic_table()) { //check column name in schema checker - if (OB_FAIL(params_.schema_checker_->check_column_exists( - params_.session_info_->get_effective_tenant_id(), table_id, col_name, is_exist))) { + if (0 == col_name.case_compare("ORA_ROWSCN") || 0 == col_name.case_compare(OB_MLOG_OLD_NEW_COLUMN_NAME)) { + //only basic table has ora_rowscn + is_exist = true; + } else if (OB_FAIL(params_.schema_checker_->check_column_exists(params_.session_info_->get_effective_tenant_id(), + table_id, + col_name, + is_exist))) { LOG_WARN("check column exists failed", K(ret)); } } else if (table_item.is_generated_table() || table_item.is_temp_table() || table_item.is_lateral_table()) { @@ -306,8 +311,7 @@ int ObColumnNamespaceChecker::check_column_exists(const TableItem &table_item, c if (OB_ISNULL(ref_stmt)) { ret = OB_NOT_INIT; LOG_WARN("generate table ref stmt is null"); - } - if (lib::is_oracle_mode() + } else if (lib::is_oracle_mode() && 0 == col_name.case_compare(OB_HIDDEN_LOGICAL_ROWID_COLUMN_NAME)) { is_exist = true; LOG_DEBUG("got rowid col when check col name, ignore check", K(ret)); @@ -561,41 +565,6 @@ bool ObColumnNamespaceChecker::hit_join_table_using_name(const JoinedTable &join return bret; } -int ObColumnNamespaceChecker::check_rowscn_table_column_namespace( - const ObQualifiedName &q_name, - const TableItem *&table_item) { - int ret = OB_SUCCESS; - table_item = nullptr; - const TableItem *cur_table = nullptr; - ObTableItemIterator table_item_iter(*this); - while (OB_SUCC(ret) - && (cur_table = table_item_iter.get_next_table_item()) != nullptr) { - if (!cur_table->is_basic_table()) { - // 兼容oracle行为，ora_rowscn视图不可见 - } else if (q_name.tbl_name_.empty()) { - if (NULL == table_item) { - table_item = cur_table; - } else { - ret = OB_NON_UNIQ_ERROR; - LOG_WARN("column in all tables is ambiguous", K(ret), K(q_name)); - } - } else { - // ora_rowscn伪列可以指定属于哪张表 - bool is_match = true; - if (OB_FAIL(ObResolverUtils::name_case_cmp(params_.session_info_, - q_name.tbl_name_, - cur_table->get_object_name(), - OB_TABLE_NAME_CLASS, - is_match))) { - LOG_WARN("table name case compare failed", K(ret)); - } else if (is_match) { - table_item = cur_table; - break; - } - } - } - return ret; -} int ObColumnNamespaceChecker::check_rowid_table_column_namespace(const ObQualifiedName &q_name, const TableItem *&table_item, bool is_from_multi_tab_insert/*default false*/) diff --git a/src/sql/resolver/dml/ob_column_namespace_checker.h b/src/sql/resolver/dml/ob_column_namespace_checker.h index a51dd2e22a..d8a4939679 100644 --- a/src/sql/resolver/dml/ob_column_namespace_checker.h +++ b/src/sql/resolver/dml/ob_column_namespace_checker.h @@ -83,10 +83,6 @@ public: join_infos_ = join_infos; } - int check_rowscn_table_column_namespace( - const ObQualifiedName &q_name, - const TableItem *&table_item); - int check_ext_table_column_namespace( const ObQualifiedName &q_name, const TableItem *&table_item); diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index 47f6a3d447..79b7c4d433 100755 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -13555,6 +13555,10 @@ int ObDMLResolver::resolve_pseudo_column( if (OB_FAIL(resolve_ora_rowscn_pseudo_column(q_name, real_ref_expr))) { LOG_WARN("resolve ora_rowscn pseudo column failed", K(ret)); } + } else if (0 == q_name.col_name_.case_compare(OB_MLOG_OLD_NEW_COLUMN_NAME)) { + if (OB_FAIL(resolve_old_new_pseudo_column(q_name, real_ref_expr))) { + LOG_WARN("resolve old_new pseudo column failed", K(ret)); + } } else if (lib::is_oracle_mode() && 0 == q_name.col_name_.case_compare(OB_HIDDEN_LOGICAL_ROWID_COLUMN_NAME)) { if (OB_FAIL(resolve_rowid_pseudo_column(q_name, real_ref_expr))) { @@ -13613,14 +13617,14 @@ int ObDMLResolver::resolve_ora_rowscn_pseudo_column( int ret = OB_SUCCESS; ObPseudoColumnRawExpr *pseudo_column_expr = NULL; const TableItem *table_item = NULL; - if (OB_FAIL(column_namespace_checker_.check_rowscn_table_column_namespace( - q_name, table_item))) { + if (OB_FAIL(column_namespace_checker_.check_table_column_namespace(q_name, table_item))) { LOG_WARN("check rowscn table colum namespace failed", K(ret)); } else if (OB_ISNULL(table_item)) { ret = OB_ERR_BAD_FIELD_ERROR; LOG_WARN("OBE_ROWSCN pseudo column only avaliable in basic table", K(ret)); - } else if (OB_FAIL(get_stmt()->get_ora_rowscn_column(table_item->table_id_, - pseudo_column_expr))) { + } else if (OB_FAIL(get_stmt()->get_target_pseudo_column(T_ORA_ROWSCN, + table_item->table_id_, + pseudo_column_expr))) { LOG_WARN("failed to get ora_rowscn column", K(ret), K(table_item)); } else if (pseudo_column_expr != NULL) { //this type of pseudo_column_expr has been add @@ -13645,6 +13649,50 @@ int ObDMLResolver::resolve_ora_rowscn_pseudo_column( return ret; } +int ObDMLResolver::resolve_old_new_pseudo_column(const ObQualifiedName &q_name, + ObRawExpr *&real_ref_expr) +{ + int ret = OB_SUCCESS; + ObPseudoColumnRawExpr *pseudo_column_expr = NULL; + const TableItem *table_item = NULL; + if (OB_ISNULL(get_stmt()) || OB_ISNULL(params_.expr_factory_) || OB_ISNULL(session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(get_stmt()), K_(params_.expr_factory), K(session_info_)); + } else if (OB_FAIL(column_namespace_checker_.check_table_column_namespace(q_name, table_item))) { + LOG_WARN("check rowscn table colum namespace failed", K(ret)); + } else if (OB_ISNULL(table_item)) { + ret = OB_ERR_BAD_FIELD_ERROR; + LOG_WARN("OBE_ROWSCN pseudo column only avaliable in basic table", K(ret)); + } else if (OB_FAIL(get_stmt()->get_target_pseudo_column(T_PSEUDO_OLD_NEW_COL, + table_item->table_id_, + pseudo_column_expr))) { + LOG_WARN("failed to get old_new column", K(ret), K(table_item)); + } else if (pseudo_column_expr != NULL) { + //this type of pseudo_column_expr has been add + real_ref_expr = pseudo_column_expr; + } else if (OB_FAIL(params_.expr_factory_->create_raw_expr(T_PSEUDO_OLD_NEW_COL, pseudo_column_expr))) { + LOG_WARN("create rowscn pseudo column expr failed", K(ret)); + } else if (OB_ISNULL(pseudo_column_expr) ) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("pseudo column expr is null", K(ret)); + } else if (OB_FAIL(pseudo_column_expr->add_relation_id(get_stmt()->get_table_bit_index(table_item->table_id_)))) { + LOG_WARN("failed to add relation id", K(ret)); + } else if (OB_FAIL(get_stmt()->get_pseudo_column_like_exprs().push_back(pseudo_column_expr))) { + LOG_WARN("fail to push back", K(ret)); + } else { + ObExprResType result_type; + result_type.set_varchar(); + result_type.set_length(1); + result_type.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_GENERAL_CI); + result_type.set_collation_level(CS_LEVEL_IMPLICIT); + pseudo_column_expr->set_result_type(result_type); + pseudo_column_expr->set_table_id(table_item->table_id_); + real_ref_expr = pseudo_column_expr; + LOG_DEBUG("old_new_expr build success", K(*pseudo_column_expr)); + } + return ret; +} + int ObDMLResolver::resolve_rowid_pseudo_column( const ObQualifiedName &q_name, ObRawExpr *&real_ref_expr) diff --git a/src/sql/resolver/dml/ob_dml_resolver.h b/src/sql/resolver/dml/ob_dml_resolver.h index 22b3164e6e..ad9c1b2417 100644 --- a/src/sql/resolver/dml/ob_dml_resolver.h +++ b/src/sql/resolver/dml/ob_dml_resolver.h @@ -899,6 +899,7 @@ private: int check_stmt_order_by(const ObSelectStmt *stmt); int resolve_ora_rowscn_pseudo_column(const ObQualifiedName &q_name, ObRawExpr *&real_ref_expr); + int resolve_old_new_pseudo_column(const ObQualifiedName &q_name, ObRawExpr *&real_ref_expr); int resolve_rowid_pseudo_column(const ObQualifiedName &q_name, ObRawExpr *&real_ref_expr); int resolve_pseudo_column(const ObQualifiedName &q_name, ObRawExpr *&real_ref_expr); int check_keystore_status(); diff --git a/src/sql/resolver/dml/ob_dml_stmt.cpp b/src/sql/resolver/dml/ob_dml_stmt.cpp index 27f4d7f1d2..510c7669b1 100644 --- a/src/sql/resolver/dml/ob_dml_stmt.cpp +++ b/src/sql/resolver/dml/ob_dml_stmt.cpp @@ -2015,13 +2015,14 @@ int ObDMLStmt::check_pseudo_column_valid() LOG_WARN("get null expr", K(ret)); } else { switch (expr->get_expr_type()) { - case T_ORA_ROWSCN: { - ObPseudoColumnRawExpr *ora_rowscn = static_cast(expr); + case T_ORA_ROWSCN: + case T_PSEUDO_OLD_NEW_COL: { + ObPseudoColumnRawExpr *pseudo_col = static_cast(expr); const TableItem *table = NULL; - if (OB_ISNULL(table = get_table_item_by_id(ora_rowscn->get_table_id())) + if (OB_ISNULL(table = get_table_item_by_id(pseudo_col->get_table_id())) || OB_UNLIKELY(!table->is_basic_table())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to find basic table for ora_rowscn", K(ret), K(table), K(*expr)); + LOG_WARN("failed to find basic table for pseudo column", K(ret), K(table), K(*expr)); } break; } @@ -4517,18 +4518,20 @@ int ObDMLStmt::collect_temp_table_infos(ObIArray &temp_table_info return ret; } -int ObDMLStmt::get_ora_rowscn_column(const uint64_t table_id, ObPseudoColumnRawExpr *&ora_rowscn) +int ObDMLStmt::get_target_pseudo_column(const ObItemType target_type, + const uint64_t table_id, + ObPseudoColumnRawExpr *&pseudo_col) { int ret = OB_SUCCESS; - ora_rowscn = NULL; + pseudo_col = NULL; ObRawExpr *expr = NULL; - for (int64_t i = 0; OB_SUCC(ret) && NULL == ora_rowscn && i < pseudo_column_like_exprs_.count(); ++i) { + for (int64_t i = 0; OB_SUCC(ret) && NULL == pseudo_col && i < pseudo_column_like_exprs_.count(); ++i) { if (OB_ISNULL(expr = pseudo_column_like_exprs_.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("expr is NULL", K(i), K(ret)); - } else if (T_ORA_ROWSCN == expr->get_expr_type() && + } else if (target_type == expr->get_expr_type() && static_cast(expr)->get_table_id() == table_id) { - ora_rowscn = static_cast(expr); + pseudo_col = static_cast(expr); } } return ret; diff --git a/src/sql/resolver/dml/ob_dml_stmt.h b/src/sql/resolver/dml/ob_dml_stmt.h index 72e33c33a3..50e2429dd7 100644 --- a/src/sql/resolver/dml/ob_dml_stmt.h +++ b/src/sql/resolver/dml/ob_dml_stmt.h @@ -875,7 +875,9 @@ public: ObSQLSessionInfo *session_info); int set_sharable_expr_reference(ObRawExpr &expr, ExplicitedRefType ref_type); int check_pseudo_column_valid(); - int get_ora_rowscn_column(const uint64_t table_id, ObPseudoColumnRawExpr *&ora_rowscn); + int get_target_pseudo_column(const ObItemType target_type, + const uint64_t table_id, + ObPseudoColumnRawExpr *&pseudo_col); virtual int remove_useless_sharable_expr(ObRawExprFactory *expr_factory, ObSQLSessionInfo *session_info, bool explicit_for_col); diff --git a/src/sql/resolver/expr/ob_expr_relation_analyzer.cpp b/src/sql/resolver/expr/ob_expr_relation_analyzer.cpp index 9a62e588b2..4ac903f6e1 100644 --- a/src/sql/resolver/expr/ob_expr_relation_analyzer.cpp +++ b/src/sql/resolver/expr/ob_expr_relation_analyzer.cpp @@ -52,7 +52,8 @@ int ObExprRelationAnalyzer::visit_expr(ObRawExpr &expr) T_PSEUDO_EXTERNAL_FILE_COL != expr.get_expr_type() && T_PSEUDO_EXTERNAL_FILE_URL != expr.get_expr_type() && T_PSEUDO_PARTITION_LIST_COL != expr.get_expr_type() && - T_ORA_ROWSCN != expr.get_expr_type()) { + T_ORA_ROWSCN != expr.get_expr_type() && + T_PSEUDO_OLD_NEW_COL != expr.get_expr_type()) { expr.get_relation_ids().reuse(); } // not sure whether we should visit onetime exec param diff --git a/src/sql/resolver/expr/ob_raw_expr.cpp b/src/sql/resolver/expr/ob_raw_expr.cpp index 754c6e5dcf..b087fa4fee 100644 --- a/src/sql/resolver/expr/ob_raw_expr.cpp +++ b/src/sql/resolver/expr/ob_raw_expr.cpp @@ -6387,6 +6387,11 @@ int ObPseudoColumnRawExpr::get_name_internal(char *buf, const int64_t buf_len, i LOG_WARN("failed to print", K(ret)); } break; + case T_PSEUDO_OLD_NEW_COL: + if (OB_FAIL(BUF_PRINTF(OB_MLOG_OLD_NEW_COLUMN_NAME))) { + LOG_WARN("failed to print", K(ret)); + } + break; case T_PSEUDO_GROUP_ID: case T_PSEUDO_STMT_ID: case T_PSEUDO_GROUP_PARAM: diff --git a/src/sql/resolver/expr/ob_raw_expr_util.cpp b/src/sql/resolver/expr/ob_raw_expr_util.cpp index 467a1c7355..94bfeb9735 100644 --- a/src/sql/resolver/expr/ob_raw_expr_util.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_util.cpp @@ -3647,6 +3647,26 @@ int ObRawExprUtils::extract_column_exprs(const ObRawExpr *raw_expr, return ret; } +int ObRawExprUtils::extract_column_exprs_and_rowscn(const ObRawExpr *raw_expr, + ObIArray &column_exprs) +{ + int ret = OB_SUCCESS; + if (NULL == raw_expr) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid raw expr", K(ret), K(raw_expr)); + } else { + if (T_REF_COLUMN == raw_expr->get_expr_type() || T_ORA_ROWSCN == raw_expr->get_expr_type()) { + ret = add_var_to_array_no_dup(column_exprs, const_cast(raw_expr)); + } else { + int64_t N = raw_expr->get_param_count(); + for (int64_t i = 0; OB_SUCC(ret) && i < N; ++i) { + ret = extract_column_exprs_and_rowscn(raw_expr->get_param_expr(i), column_exprs); + } + } + } + return ret; +} + int ObRawExprUtils::extract_contain_exprs(ObRawExpr *raw_expr, const common::ObIArray &src_exprs, common::ObIArray &contain_exprs) diff --git a/src/sql/resolver/expr/ob_raw_expr_util.h b/src/sql/resolver/expr/ob_raw_expr_util.h index c7db93028a..7cfbc562fa 100644 --- a/src/sql/resolver/expr/ob_raw_expr_util.h +++ b/src/sql/resolver/expr/ob_raw_expr_util.h @@ -417,6 +417,8 @@ public: static int extract_column_exprs(const ObRawExpr *raw_expr, common::ObIArray &column_exprs, bool need_pseudo_column = false); + static int extract_column_exprs_and_rowscn(const ObRawExpr *raw_expr, + common::ObIArray &column_exprs); static int extract_column_exprs(const common::ObIArray &exprs, common::ObIArray &column_exprs, bool need_pseudo_column = false); diff --git a/src/sql/resolver/mv/ob_mv_checker.cpp b/src/sql/resolver/mv/ob_mv_checker.cpp index a2cdfa58bd..e6b365b229 100644 --- a/src/sql/resolver/mv/ob_mv_checker.cpp +++ b/src/sql/resolver/mv/ob_mv_checker.cpp @@ -24,13 +24,16 @@ using namespace common; namespace sql { -int ObMVChecker::check_mv_fast_refresh_valid(const ObSelectStmt *view_stmt, - ObStmtFactory *stmt_factory, - ObRawExprFactory *expr_factory, - ObSQLSessionInfo *session_info) +int ObMVChecker::check_mv_fast_refresh_type(const ObSelectStmt *view_stmt, + ObStmtFactory *stmt_factory, + ObRawExprFactory *expr_factory, + ObSQLSessionInfo *session_info, + ObTableSchema &container_table_schema, + ObMVRefreshableType &refresh_type) { int ret = OB_SUCCESS; ObDMLStmt *copied_stmt = NULL; + refresh_type = OB_MV_REFRESH_INVALID; if (OB_ISNULL(view_stmt) || OB_ISNULL(stmt_factory) || OB_ISNULL(expr_factory) || OB_ISNULL(session_info)) { ret = OB_ERR_UNEXPECTED; @@ -42,13 +45,14 @@ int ObMVChecker::check_mv_fast_refresh_valid(const ObSelectStmt *view_stmt, LOG_WARN("failed to formalize stmt reference", K(ret)); } else { FastRefreshableNotes notes; - ObMVChecker checker(*static_cast(copied_stmt), *expr_factory, session_info); + ObMVChecker checker(*static_cast(copied_stmt), *expr_factory, session_info, + container_table_schema); checker.set_fast_refreshable_note(¬es); if (OB_FAIL(checker.check_mv_refresh_type())) { LOG_WARN("failed to check mv refresh type", K(ret)); - } else if (OB_UNLIKELY(OB_MV_FAST_REFRESH_SIMPLE_MAV > checker.get_refersh_type())) { - ret = OB_ERR_MVIEW_CAN_NOT_FAST_REFRESH; - LOG_WARN("fast refresh is not supported for this mv", K(ret), K(checker.get_refersh_type()), K(notes)); + } else { + refresh_type = checker.get_refersh_type(); + LOG_INFO("check mv fast refresh type", KR(ret), K(refresh_type), K(notes)); } } return ret; @@ -134,10 +138,10 @@ int ObMVChecker::check_mv_stmt_refresh_type_basic(const ObSelectStmt &stmt, bool } if (OB_SUCC(ret)) { - bool mlog_valid = false; - if (OB_FAIL(check_mv_dependency_mlog_tables(stmt, mlog_valid))) { + bool table_type_valid = false; + if (OB_FAIL(check_mv_table_type_valid(stmt, table_type_valid))) { LOG_WARN("failed to check mv table mlog", K(ret)); - } else if (!mlog_valid) { + } else if (!table_type_valid) { is_valid = false; } } @@ -183,6 +187,38 @@ int ObMVChecker::check_mv_duplicated_exprs(const ObSelectStmt &stmt, bool &has_d return ret; } +int ObMVChecker::check_mv_table_type_valid(const ObSelectStmt &stmt, bool &is_valid) +{ + int ret = OB_SUCCESS; + is_valid = false; + mlog_tables_.reuse(); + ObSqlSchemaGuard *sql_schema_guard = NULL; + if (OB_ISNULL(stmt.get_query_ctx()) + || OB_ISNULL(sql_schema_guard = &stmt.get_query_ctx()->sql_schema_guard_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(sql_schema_guard)); + } else if (stmt.get_table_size() == 0) { + is_valid = false; + } else { + is_valid = true; + const ObIArray &tables = stmt.get_table_items(); + const TableItem *table = NULL; + for (int64_t i = 0; is_valid && OB_SUCC(ret) && i < tables.count(); ++i) { + if (OB_ISNULL(table = tables.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected table", K(ret), KPC(table)); + } else if (OB_UNLIKELY(!table->is_basic_table())) { + is_valid = false; + append_fast_refreshable_note("basic table allowed only"); + } else if (OB_NOT_NULL(table->flashback_query_expr_)) { + is_valid = false; + append_fast_refreshable_note("flashback query not support"); + } + } + } + return ret; +} + int ObMVChecker::check_mv_dependency_mlog_tables(const ObSelectStmt &stmt, bool &is_valid) { int ret = OB_SUCCESS; @@ -205,12 +241,6 @@ int ObMVChecker::check_mv_dependency_mlog_tables(const ObSelectStmt &stmt, bool if (OB_ISNULL(table = tables.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected table", K(ret), KPC(table)); - } else if (OB_UNLIKELY(!table->is_basic_table())) { - is_valid = false; - append_fast_refreshable_note("basic table allowed only"); - } else if (OB_NOT_NULL(table->flashback_query_expr_)) { - is_valid = false; - append_fast_refreshable_note("flashback query not support"); } else if (OB_FAIL(sql_schema_guard->get_table_schema(table->ref_id_, table_schema))) { LOG_WARN("failed to get table schema", K(ret)); } else if (OB_ISNULL(table_schema)) { @@ -331,7 +361,9 @@ int ObMVChecker::check_mav_refresh_type_basic(const ObSelectStmt &stmt, bool &is LOG_WARN("failed to check is standard group by", K(ret)); } else if (!is_valid) { append_fast_refreshable_note("select output is not standard group by for mysql mode", OB_MV_FAST_REFRESH_SIMPLE_MAV); - } else { + } else if (OB_FAIL(check_mv_dependency_mlog_tables(stmt, is_valid))) { + LOG_WARN("failed to check mv table mlog", KR(ret)); + } else if (is_valid) { // check group by exprs exists in select list const ObIArray &group_exprs = stmt.get_group_exprs(); for (int64_t i = 0; is_valid && OB_SUCC(ret) && i < group_exprs.count(); ++i) { @@ -637,12 +669,25 @@ int ObMVChecker::check_mjv_refresh_type(const ObSelectStmt &stmt, ObMVRefreshabl { int ret = OB_SUCCESS; refresh_type = OB_MV_COMPLETE_REFRESH; + bool mlog_valid = true; + bool match_major_refresh = false; bool is_valid = false; - if (OB_FAIL(check_join_mv_fast_refresh_valid(stmt, false, is_valid))) { + uint64_t tenant_id = MTL_ID(); + uint64_t data_version = 0; + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { + LOG_WARN("fail to get tenant data version", KR(ret), K(data_version)); + } else if (OB_FAIL(check_join_mv_fast_refresh_valid(stmt, false, is_valid))) { LOG_WARN("failed to check join mv fast refresh valid", K(ret)); } else if (!is_valid) { /* do nothing */ - } else { + } else if (data_version >= DATA_VERSION_4_3_4_0 && + OB_FAIL(check_match_major_refresh_mv(stmt, match_major_refresh))) { + LOG_WARN("failed to check match major refresh mv", KR(ret)); + } else if (match_major_refresh) { + refresh_type = OB_MV_FAST_REFRESH_MAJOR_REFRESH_MJV; + } else if (OB_FAIL(check_mv_dependency_mlog_tables(stmt, mlog_valid))) { + LOG_WARN("failed to check mv dependency mlog tables", KR(ret)); + } else if (mlog_valid) { refresh_type = OB_MV_FAST_REFRESH_SIMPLE_MJV; } return ret; @@ -657,6 +702,7 @@ int ObMVChecker::check_join_mv_fast_refresh_valid(const ObSelectStmt &stmt, bool join_type_valid = false; bool all_table_exists_rowkey = false; bool select_valid = false; + if (stmt.get_table_size() <= 1) { append_fast_refreshable_note("table size not support"); // } else if (stmt.get_table_size() > 5) { @@ -801,5 +847,244 @@ void ObMVChecker::append_fast_refreshable_note(const char *str, } } -}//end of namespace sql +int ObMVChecker::check_match_major_refresh_mv(const ObSelectStmt &stmt, bool &is_match) +{ + int ret = OB_SUCCESS; + const ObIArray &joined_tables = stmt.get_joined_tables(); + const JoinedTable *joined_table = NULL; + const ObTableSchema *left_table_schema = NULL; + const ObTableSchema *right_table_schema = NULL; + ObSqlSchemaGuard &sql_schema_guard = stmt.get_query_ctx()->sql_schema_guard_; + is_match = true; + + if (stmt.get_table_size() != 2 || joined_tables.count() != 1) { + is_match = false; + LOG_INFO("[MAJ_REF_MV] join table size not valid", KR(ret), "table_size", stmt.get_table_size(), + "joined_table_count", joined_tables.count()); + } else if (FALSE_IT(joined_table = joined_tables.at(0))) { + + } else if (OB_ISNULL(joined_table) || + OB_ISNULL(joined_table->left_table_) || OB_ISNULL(joined_table->right_table_)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("joined_table is null", KR(ret)); + } else if (OB_FAIL(sql_schema_guard.get_table_schema(joined_table->left_table_->ref_id_, + left_table_schema))) { + LOG_WARN("left table schema not found", KR(ret), KPC(joined_table->left_table_)); + } else if (OB_FAIL(sql_schema_guard.get_table_schema(joined_table->right_table_->ref_id_, + right_table_schema))) { + LOG_WARN("right table schema not found", KR(ret), KPC(joined_table->right_table_)); + } else if (OB_ISNULL(left_table_schema) || OB_ISNULL(right_table_schema)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("table schema is null", KR(ret), K(left_table_schema), K(right_table_schema)); + } else if (OB_FAIL(check_right_table_join_key_valid(stmt, joined_table, right_table_schema, + is_match))) { + LOG_WARN("failed to check join key valid", KR(ret)); + } else if (is_match && OB_FAIL(check_left_table_partition_rule_valid( + stmt, joined_table->left_table_, left_table_schema, is_match))) { + LOG_WARN("failed to check partition rule valid", KR(ret)); + } else if (is_match && OB_FAIL(check_left_table_rowkey_valid(stmt, left_table_schema, is_match))) { + LOG_WARN("failed to check left table rowkey valid", KR(ret)); + } else if (is_match && OB_FAIL(check_broadcast_table_valid(stmt, right_table_schema, is_match))) { + LOG_WARN("failed to check broadcast table valid", KR(ret)); + } else if (is_match && OB_FAIL(check_column_store_valid(stmt, is_match))) { + LOG_WARN("failed to check column store valid", KR(ret)); + } + + LOG_INFO("[MAJ_REF_MV] check match major refresh mv", K(is_match)); + + return ret; +} + +int ObMVChecker::check_right_table_join_key_valid(const ObSelectStmt &stmt, + const JoinedTable *joined_table, + const ObTableSchema *right_table_schema, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObRelIds right_table_set; + is_valid = true; + + if (OB_ISNULL(joined_table) || OB_ISNULL(joined_table->right_table_) || OB_ISNULL(right_table_schema)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("null input", KR(ret), K(joined_table), K(right_table_schema)); + } else if (OB_FAIL(stmt.get_table_rel_ids(*joined_table->right_table_, right_table_set))) { + LOG_WARN("failed to get table rel ids", KR(ret)); + } else { + const ObIArray &join_conditions = joined_table->get_join_conditions(); + ObSEArray right_table_keys; + ObSEArray right_table_join_keys; + for (int64_t i = 0; OB_SUCC(ret) && i < join_conditions.count(); ++i) { + const ObRawExpr *expr = join_conditions.at(i); + const ObRawExpr *l_expr = NULL; + const ObRawExpr *r_expr = NULL; + if (OB_ISNULL(expr)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("expr is null", KR(ret)); + } else if (2 != expr->get_param_count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect expr param count", KR(ret), K(expr->get_param_count())); + } else if (OB_ISNULL(l_expr = expr->get_param_expr(0)) || + OB_ISNULL(r_expr = expr->get_param_expr(1))) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("expr is null", KR(ret), K(l_expr), K(r_expr)); + } // TODO: expr's table_id is not valid, we can only compare table_name. maybe set and check table_id later + else if (l_expr->is_column_ref_expr() && + l_expr->get_relation_ids().is_subset(right_table_set) && + OB_FAIL(right_table_join_keys.push_back( + static_cast(l_expr)->get_column_id()))) { + LOG_WARN("failed to push join key", KR(ret)); + } else if (r_expr->is_column_ref_expr() && + r_expr->get_relation_ids().is_subset(right_table_set) && + OB_FAIL(right_table_join_keys.push_back( + static_cast(r_expr)->get_column_id()))) { + LOG_WARN("failed to push join key", KR(ret)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(right_table_schema->get_rowkey_info().get_column_ids(right_table_keys))) { + LOG_WARN("failed to get table keys", KR(ret)); + } else { + lib::ob_sort(right_table_keys.begin(), right_table_keys.end()); + lib::ob_sort(right_table_join_keys.begin(), right_table_join_keys.end()); + if (!is_array_equal(right_table_keys, right_table_join_keys)) { + is_valid = false; + LOG_INFO("[MAJ_REF_MV] right table join key is not valid", K(right_table_keys), + K(right_table_join_keys)); + } + } + } + } + + return ret; +} + +int ObMVChecker::check_left_table_partition_rule_valid(const ObSelectStmt &stmt, + const TableItem *left_table, + const ObTableSchema *left_table_schema, + bool &is_valid) +{ + int ret = OB_SUCCESS; + ObRelIds left_table_set; + is_valid = true; + + if (OB_ISNULL(left_table) || OB_ISNULL(left_table_schema)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("left table schema is null", KR(ret), K(left_table), K(left_table_schema)); + } else if (OB_FAIL(stmt.get_table_rel_ids(*left_table, left_table_set))) { + LOG_WARN("failed to get table rel ids", KR(ret)); + } else { + const schema::ObPartitionOption &left_partop = left_table_schema->get_part_option(); + const schema::ObPartitionOption &mv_partop = mv_container_table_schema_.get_part_option(); + // TODO for now only works for collect_mv, we could relax the rule later: partition type can + // be other type, partition func expr should consider about column name alias, etc. + if (!left_partop.is_hash_part() || !mv_partop.is_hash_part()) { + is_valid = false; + LOG_INFO("[MAJ_REF_MV] is not hash partition", K(left_partop), K(mv_partop)); + } else if (left_partop.get_part_num() != mv_partop.get_part_num()) { + is_valid = false; + LOG_INFO("[MAJ_REF_MV] part num doesn't match", K(left_partop), K(mv_partop)); + } else { + ObString left_part_str = left_partop.get_part_func_expr_str(); + ObString mv_part_str = mv_partop.get_part_func_expr_str(); + bool found_item = false; + for (int64_t i = 0; OB_SUCC(ret) && i < stmt.get_select_items().count(); ++i) { + const ObRawExpr *expr = NULL; + int64_t idx = OB_INVALID_INDEX; + if (OB_ISNULL(expr = stmt.get_select_items().at(i).expr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", KR(ret), K(i), K(stmt.get_select_items())); + } else if (!expr->is_column_ref_expr() || + !expr->get_relation_ids().is_subset(left_table_set)) { + } else { + const ObColumnRefRawExpr *col_expr = static_cast(expr); + if (!col_expr->get_alias_column_name().empty() && + mv_part_str.case_compare(col_expr->get_alias_column_name()) != 0) { + } else if (col_expr->get_alias_column_name().empty() && + mv_part_str.case_compare(col_expr->get_column_name()) != 0) { + } else if (left_part_str.case_compare(col_expr->get_column_name()) != 0) { + } else { + found_item = true; + break; + } + LOG_INFO("iter select item", K(*col_expr)); + } + } + if (OB_FAIL(ret)) { + } else if (!found_item) { + is_valid = false; + LOG_INFO("[MAJ_REF_MV] hash expr doesn't match", K(left_partop), K(mv_partop)); + } + } + } + + return ret; +} + +int ObMVChecker::check_left_table_rowkey_valid(const ObSelectStmt &stmt, + const ObTableSchema *left_table_schema, + bool &is_valid) +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(left_table_schema)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("left table schema is null", KR(ret), K(left_table_schema)); + } else { + common::ObArray left_table_rowkey_column_ids; + common::ObArray mv_table_rowkey_column_ids; + if (OB_FAIL(left_table_schema->get_rowkey_info().get_column_ids(left_table_rowkey_column_ids))) { + LOG_WARN("failed to get table keys", KR(ret)); + } else if (OB_FAIL(mv_container_table_schema_.get_rowkey_info().get_column_ids( + mv_table_rowkey_column_ids))) { + LOG_WARN("failed to get table keys", KR(ret)); + } else { + lib::ob_sort(left_table_rowkey_column_ids.begin(), left_table_rowkey_column_ids.end()); + lib::ob_sort(mv_table_rowkey_column_ids.begin(), mv_table_rowkey_column_ids.end()); + if (!is_array_equal(left_table_rowkey_column_ids, mv_table_rowkey_column_ids)) { + is_valid = false; + LOG_INFO("[MAJ_REF_MV] rowkey doesn't match", K(left_table_rowkey_column_ids), + K(mv_table_rowkey_column_ids)); + } + } + } + + return ret; +} + +int ObMVChecker::check_broadcast_table_valid(const ObSelectStmt &stmt, + const ObTableSchema *right_table_schema, + bool &is_valid) +{ + int ret = OB_SUCCESS; + is_valid = true; + + if (OB_ISNULL(right_table_schema)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("right table schema is null", KR(ret)); + } else if (!right_table_schema->is_broadcast_table()) { + is_valid = false; + LOG_INFO("[MAJ_REF_MV] right table is not broadcast table", K(*right_table_schema)); + } + + return ret; +} + +int ObMVChecker::check_column_store_valid(const ObSelectStmt &stmt, bool &is_valid) +{ + int ret = OB_SUCCESS; + bool is_column_store = false; + is_valid = true; + + if (OB_FAIL(mv_container_table_schema_.get_is_column_store(is_column_store))) { + LOG_WARN("failed to get is column store", KR(ret)); + } else if (is_column_store) { + is_valid = false; + LOG_INFO("[MAJ_REF_MV] mv container table is column store table", + K(mv_container_table_schema_.get_table_name())); + } + + return ret; +} + +} // end of namespace sql }//end of namespace oceanbase diff --git a/src/sql/resolver/mv/ob_mv_checker.h b/src/sql/resolver/mv/ob_mv_checker.h index 979d7a9b67..1c050ad695 100644 --- a/src/sql/resolver/mv/ob_mv_checker.h +++ b/src/sql/resolver/mv/ob_mv_checker.h @@ -32,6 +32,10 @@ enum ObMVRefreshableType OB_MV_FAST_REFRESH_SIMPLE_JOIN_MAV, // fast refresh for inner join MAV OB_MV_FAST_REFRESH_MAJOR_REFRESH_MJV, // fast refresh for major compaction mv }; +inline bool IS_VALID_FAST_REFRESH_TYPE(ObMVRefreshableType type) +{ + return type >= OB_MV_FAST_REFRESH_SIMPLE_MAV; +} struct FastRefreshableNotes { @@ -55,24 +59,29 @@ class ObMVChecker public: explicit ObMVChecker(const ObSelectStmt &stmt, ObRawExprFactory &expr_factory, - ObSQLSessionInfo *session_info) + ObSQLSessionInfo *session_info, + const ObTableSchema &mv_container_table_schema) : stmt_(stmt), refresh_type_(OB_MV_REFRESH_INVALID), expr_factory_(expr_factory), session_info_(session_info), - fast_refreshable_note_(NULL) + fast_refreshable_note_(NULL), + mv_container_table_schema_(mv_container_table_schema) {} ~ObMVChecker() {} - static int check_mv_fast_refresh_valid(const ObSelectStmt *view_stmt, - ObStmtFactory *stmt_factory, - ObRawExprFactory *expr_factory, - ObSQLSessionInfo *session_info); + static int check_mv_fast_refresh_type(const ObSelectStmt *view_stmt, + ObStmtFactory *stmt_factory, + ObRawExprFactory *expr_factory, + ObSQLSessionInfo *session_info, + ObTableSchema &container_table_schema, + ObMVRefreshableType &refresh_type); int check_mv_refresh_type(); ObMVRefreshableType get_refersh_type() const { return refresh_type_; }; static bool is_basic_aggr(const ObItemType aggr_type); static int get_dependent_aggr_of_fun_sum(const ObSelectStmt &stmt, const ObRawExpr *sum_param, const ObAggFunRawExpr *&dep_aggr); const ObSelectStmt &get_stmt() const { return stmt_; } + const ObTableSchema &get_mv_container_table_schema() const { return mv_container_table_schema_; } const ObIArray> &get_expand_aggrs() const { return expand_aggrs_; } int get_mlog_table_schema(const TableItem *table, const share::schema::ObTableSchema *&mlog_schema) const; void set_fast_refreshable_note(FastRefreshableNotes *note) { fast_refreshable_note_ = note; } @@ -83,6 +92,8 @@ private: int check_select_contains_all_tables_primary_key(const ObSelectStmt &stmt, bool &all_table_exists_rowkey, bool &contain_all_rowkey); + int check_mv_stmt_use_special_expr(const ObSelectStmt &stmt, bool &has_special_expr); + int check_mv_table_type_valid(const ObSelectStmt &stmt, bool &is_valid); int check_mv_dependency_mlog_tables(const ObSelectStmt &stmt, bool &is_valid); int check_mv_duplicated_exprs(const ObSelectStmt &stmt, bool &has_dup_exprs); bool check_mlog_table_valid(const share::schema::ObTableSchema *table_schema, @@ -118,6 +129,16 @@ private: const bool for_join_mav, bool &is_valid); void append_fast_refreshable_note(const char *str, const ObMVRefreshableType type = OB_MV_COMPLETE_REFRESH); + int check_match_major_refresh_mv(const ObSelectStmt &stmt, bool &is_match); + int check_right_table_join_key_valid(const ObSelectStmt &stmt, const JoinedTable *joined_table, + const ObTableSchema *right_table_schema, bool &is_valid); + int check_left_table_partition_rule_valid(const ObSelectStmt &stmt, const TableItem *left_table, + const ObTableSchema *left_table_schema, bool &is_valid); + int check_left_table_rowkey_valid(const ObSelectStmt &stmt, + const ObTableSchema *left_table_schema, bool &is_valid); + int check_broadcast_table_valid(const ObSelectStmt &stmt, const ObTableSchema *right_table_schema, + bool &is_valid); + int check_column_store_valid(const ObSelectStmt &stmt, bool &is_valid); const ObSelectStmt &stmt_; ObMVRefreshableType refresh_type_; @@ -127,6 +148,7 @@ private: ObRawExprFactory &expr_factory_; ObSQLSessionInfo *session_info_; FastRefreshableNotes *fast_refreshable_note_; + const ObTableSchema &mv_container_table_schema_; DISALLOW_COPY_AND_ASSIGN(ObMVChecker); }; diff --git a/src/sql/resolver/mv/ob_mv_dep_utils.cpp b/src/sql/resolver/mv/ob_mv_dep_utils.cpp index e8afdddb44..05a15bab1f 100644 --- a/src/sql/resolver/mv/ob_mv_dep_utils.cpp +++ b/src/sql/resolver/mv/ob_mv_dep_utils.cpp @@ -248,5 +248,107 @@ int ObMVDepUtils::get_table_ids_only_referenced_by_given_mv( } return ret; } + +int ObMVDepUtils::get_table_ids_only_referenced_by_given_fast_lsm_mv( + ObISQLClient &sql_client, + const uint64_t tenant_id, + const uint64_t mview_table_id, + ObIArray &ref_table_ids) +{ + int ret = OB_SUCCESS; + if ((OB_INVALID_TENANT_ID == tenant_id) + || (OB_INVALID_ID == mview_table_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant_id or mview_table_id", + KR(ret), K(tenant_id), K(mview_table_id)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + ObSqlString sql; + ObMySQLResult *result = NULL; + const uint64_t exec_tenant_id = ObSchemaUtils::get_exec_tenant_id(tenant_id); + if (OB_FAIL(sql.assign_fmt( + "select a.p_obj from" + " (select p_obj from %s dep, %s mv where dep.mview_id = mv.mview_id and " + "mv.refresh_mode in (%ld) group by p_obj having count(*) = 1) a," + " (select p_obj from %s dep, %s mv where dep.mview_id = mv.mview_id and " + "mv.refresh_mode in (%ld) and dep.tenant_id = %lu and dep.mview_id = %lu) b " + "where a.p_obj = b.p_obj", + OB_ALL_MVIEW_DEP_TNAME, OB_ALL_MVIEW_TNAME, + ObMVRefreshMode::MAJOR_COMPACTION, + OB_ALL_MVIEW_DEP_TNAME, OB_ALL_MVIEW_TNAME, + ObMVRefreshMode::MAJOR_COMPACTION, + ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, tenant_id), mview_table_id))) { + LOG_WARN("failed to assign sql", KR(ret)); + } else if (OB_FAIL(sql_client.read(res, tenant_id, sql.ptr()))) { + LOG_WARN("failed to execute read", KR(ret), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is null", KR(ret), KP(result)); + } else { + while (OB_SUCC(ret)) { + if (OB_FAIL(result->next())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get next", KR(ret)); + } else { + ret = OB_SUCCESS; + break; + } + } else { + uint64_t ref_table_id = OB_INVALID_ID; + EXTRACT_INT_FIELD_MYSQL(*result, "p_obj", ref_table_id, uint64_t); + if (OB_SUCC(ret)) { + if (OB_FAIL(ref_table_ids.push_back(ref_table_id))) { + LOG_WARN("failed to add ref table id to array", KR(ret), K(ref_table_id)); + } + } + } + } + } + } + } + return ret; +} +int ObMVDepUtils::get_referring_mv_of_base_table(ObISQLClient &sql_client, const uint64_t tenant_id, + const uint64_t base_table_id, + ObIArray &mview_ids) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + + SMART_VAR(ObMySQLProxy::MySQLResult, res) + { + ObMySQLResult *result = nullptr; + if (OB_FAIL(sql.assign_fmt("SELECT mview_id FROM %s WHERE p_obj = %ld", + share::OB_ALL_MVIEW_DEP_TNAME, base_table_id))) { + LOG_WARN("fail to assign sql", KR(ret)); + } else if (OB_FAIL(sql_client.read(res, tenant_id, sql.ptr()))) { + LOG_WARN("execute sql failed", KR(ret), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result is null", KR(ret)); + } else { + while (OB_SUCC(ret)) { + if (OB_FAIL(result->next())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get next", KR(ret)); + } else { + ret = OB_SUCCESS; + break; + } + } else { + uint64_t mview_id = 0; + EXTRACT_INT_FIELD_MYSQL(*result, "mview_id", mview_id, uint64_t); + if (OB_SUCC(ret)) { + if (OB_FAIL(mview_ids.push_back(mview_id))) { + LOG_WARN("failed to add ref table id to array", KR(ret), K(mview_id)); + } + } + } + } + } + } + + return ret; +} } // end of sql } // end of oceanbase \ No newline at end of file diff --git a/src/sql/resolver/mv/ob_mv_dep_utils.h b/src/sql/resolver/mv/ob_mv_dep_utils.h index 9963a22ac0..d1e70dec3f 100644 --- a/src/sql/resolver/mv/ob_mv_dep_utils.h +++ b/src/sql/resolver/mv/ob_mv_dep_utils.h @@ -76,6 +76,14 @@ public: const uint64_t tenant_id, const uint64_t mview_table_id, common::ObIArray &ref_table_ids); + static int get_table_ids_only_referenced_by_given_fast_lsm_mv( + common::ObISQLClient &sql_client, + const uint64_t tenant_id, + const uint64_t mview_table_id, + common::ObIArray &ref_table_ids); + static int get_referring_mv_of_base_table(ObISQLClient &sql_client, const uint64_t tenant_id, + const uint64_t base_table_id, + ObIArray &mview_ids); }; } // end of sql } // end of oceanbase diff --git a/src/sql/resolver/mv/ob_mv_printer.cpp b/src/sql/resolver/mv/ob_mv_printer.cpp index 50719217b3..0d8c6844b5 100644 --- a/src/sql/resolver/mv/ob_mv_printer.cpp +++ b/src/sql/resolver/mv/ob_mv_printer.cpp @@ -36,10 +36,12 @@ const ObString ObMVPrinter::WIN_MAX_SEQ_COL_NAME = "MAXSEQ$$"; const ObString ObMVPrinter::WIN_MIN_SEQ_COL_NAME = "MINSEQ$$"; int ObMVPrinter::print_mv_operators(const share::schema::ObTableSchema &mv_schema, + const share::schema::ObTableSchema &mv_container_schema, const ObSelectStmt &view_stmt, const bool for_rt_expand, const share::SCN &last_refresh_scn, const share::SCN &refresh_scn, + const MajorRefreshInfo *major_refresh_info, ObIAllocator &alloc, ObIAllocator &str_alloc, ObSchemaGetterGuard *schema_guard, @@ -52,7 +54,7 @@ int ObMVPrinter::print_mv_operators(const share::schema::ObTableSchema &mv_schem int ret = OB_SUCCESS; operators.reuse(); refreshable_type = OB_MV_REFRESH_INVALID; - ObMVChecker checker(view_stmt, expr_factory, session_info); + ObMVChecker checker(view_stmt, expr_factory, session_info, mv_container_schema); ObMVPrinter printer(alloc, mv_schema, checker, for_rt_expand, stmt_factory, expr_factory); ObSEArray dml_stmts; if (OB_ISNULL(view_stmt.get_query_ctx())) { @@ -62,7 +64,7 @@ int ObMVPrinter::print_mv_operators(const share::schema::ObTableSchema &mv_schem LOG_WARN("failed to check mv refresh type", K(ret)); } else if (OB_MV_COMPLETE_REFRESH >= (refreshable_type = checker.get_refersh_type())) { LOG_TRACE("mv not support fast refresh", K(refreshable_type), K(mv_schema.get_table_name())); - } else if (OB_FAIL(printer.init(last_refresh_scn, refresh_scn))) { + } else if (OB_FAIL(printer.init(last_refresh_scn, refresh_scn, major_refresh_info))) { LOG_WARN("failed to init mv printer", K(ret)); } else if (OB_FAIL(printer.gen_mv_operator_stmts(dml_stmts))) { LOG_WARN("failed to print mv operators", K(ret)); @@ -139,6 +141,12 @@ int ObMVPrinter::gen_refresh_dmls_for_mv(ObIArray &dml_stmts) } break; } + case OB_MV_FAST_REFRESH_MAJOR_REFRESH_MJV: { + if (OB_FAIL(gen_refresh_select_for_major_refresh_mjv(dml_stmts))) { + LOG_WARN("fail to gen refresh select for major refresh mjv", K(ret)); + } + break; + } default: { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected refresh type", K(ret), K(mv_checker_.get_refersh_type())); @@ -165,6 +173,12 @@ int ObMVPrinter::gen_real_time_view_for_mv(ObSelectStmt *&sel_stmt) } break; } + case OB_MV_FAST_REFRESH_MAJOR_REFRESH_MJV: { + if (OB_FAIL(gen_real_time_view_for_major_refresh_mjv(sel_stmt))) { + LOG_WARN("fail to gen real time view for simple mjv", K(ret)); + } + break; + } default: { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected mv type for real-time mview", K(ret), K(mv_checker_.get_refersh_type())); @@ -1730,13 +1744,15 @@ int ObMVPrinter::create_simple_table_item(ObDMLStmt *stmt, } int ObMVPrinter::init(const share::SCN &last_refresh_scn, - const share::SCN &refresh_scn) + const share::SCN &refresh_scn, + const MajorRefreshInfo *major_refresh_info) { int ret = OB_SUCCESS; inited_ = false; ObCollationType cs_type = ObCharset::get_default_collation(ObCharset::get_default_charset()); ObQueryCtx *query_ctx = NULL; const ObDatabaseSchema *db_schema = NULL; + major_refresh_info_ = major_refresh_info; if (OB_ISNULL(query_ctx = stmt_factory_.get_query_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret), K(query_ctx)); @@ -1910,7 +1926,8 @@ int ObMVPrinter::gen_exists_cond_for_table(const TableItem *source_table, const TableItem *outer_table, const bool is_exists, const bool use_orig_sel_alias, - ObRawExpr *&exists_expr) + ObRawExpr *&exists_expr, + bool use_mlog /* default true */) { int ret = OB_SUCCESS; exists_expr = NULL; @@ -1918,7 +1935,7 @@ int ObMVPrinter::gen_exists_cond_for_table(const TableItem *source_table, ObQueryRefRawExpr *query_ref_expr = NULL; const ObTableSchema *mlog_schema = NULL; ObSelectStmt *subquery = NULL; - TableItem *mlog_table = NULL; + TableItem *delta_src_table = NULL; SelectItem sel_item; ObSEArray rowkey_column_ids; const ObTableSchema *source_table_schema = NULL; @@ -1926,17 +1943,17 @@ int ObMVPrinter::gen_exists_cond_for_table(const TableItem *source_table, if (OB_ISNULL(outer_table) || OB_ISNULL(source_table) || OB_ISNULL(stmt_factory_.get_query_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret), K(outer_table), K(source_table), K(stmt_factory_.get_query_ctx())); - } else if (OB_FAIL(mv_checker_.get_mlog_table_schema(source_table, mlog_schema))) { + } else if (use_mlog && OB_FAIL(mv_checker_.get_mlog_table_schema(source_table, mlog_schema))) { LOG_WARN("failed to get mlog schema", K(ret), KPC(source_table)); } else if (OB_FAIL(stmt_factory_.get_query_ctx()->sql_schema_guard_.get_table_schema(source_table->ref_id_, source_table_schema))) { LOG_WARN("failed to get table schema", K(ret)); } else if (OB_FAIL(expr_factory_.create_raw_expr(T_REF_QUERY, query_ref_expr)) || OB_FAIL(expr_factory_.create_raw_expr(is_exists ? T_OP_EXISTS : T_OP_NOT_EXISTS, exists_op_expr))) { LOG_WARN("failed to create raw expr", K(ret)); - } else if (OB_ISNULL(query_ref_expr) || OB_ISNULL(exists_op_expr) || OB_ISNULL(mlog_schema) - || OB_ISNULL(source_table_schema)) { + } else if (OB_ISNULL(query_ref_expr) || OB_ISNULL(exists_op_expr) + || OB_UNLIKELY(use_mlog && NULL == mlog_schema)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected NULL", K(ret), K(query_ref_expr), K(exists_op_expr), K(mlog_schema), K(source_table_schema)); + LOG_WARN("unexpected NULL", K(ret), K(query_ref_expr), K(exists_op_expr), K(use_mlog), K(mlog_schema)); } else if (OB_UNLIKELY(source_table_schema->is_heap_table())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret), K(source_table_schema)); @@ -1944,17 +1961,26 @@ int ObMVPrinter::gen_exists_cond_for_table(const TableItem *source_table, LOG_WARN("failed to add param expr", K(ret)); } else if (OB_FAIL(create_simple_stmt(subquery))) { LOG_WARN("failed to create simple stmt", K(ret)); - } else if (OB_FAIL(create_simple_table_item(subquery, mlog_schema->get_table_name_str(), mlog_table))) { + } else if (OB_FAIL(create_simple_table_item(subquery, (use_mlog ? mlog_schema->get_table_name_str() : source_table->table_name_), + delta_src_table))) { LOG_WARN("failed to create simple table item", K(ret)); } else if (OB_FAIL(subquery->get_select_items().push_back(sel_item))) { LOG_WARN("failed to push back not exists expr", K(ret)); - } else if (OB_FAIL(gen_delta_table_view_conds(*mlog_table, subquery->get_condition_exprs()))) { + } else if (OB_FAIL(gen_delta_table_view_conds(*delta_src_table, subquery->get_condition_exprs()))) { LOG_WARN("failed to generate delta table view conds", K(ret)); + } else if (!use_mlog && OB_FAIL(append_old_new_col_filter(*delta_src_table, + exprs_.str_o_, + subquery->get_condition_exprs()))) { + LOG_WARN("failed to generate old new filter", K(ret)); } else if (OB_FAIL(source_table_schema->get_rowkey_column_ids(rowkey_column_ids))) { LOG_WARN("failed to get rowkey column ids", KR(ret)); } else { exists_expr = exists_op_expr; - mlog_table->database_name_ = source_table->database_name_; + delta_src_table->database_name_ = source_table->database_name_; + if (!for_rt_expand_ && !use_mlog) { + delta_src_table->flashback_query_expr_ = exprs_.refresh_scn_; + delta_src_table->flashback_query_type_ = TableItem::USING_SCN; + } query_ref_expr->set_ref_stmt(subquery); const ObColumnSchemaV2 *rowkey_column = NULL; ObRawExpr *inner_expr = NULL; @@ -1971,7 +1997,7 @@ int ObMVPrinter::gen_exists_cond_for_table(const TableItem *source_table, rowkey_column_ids.at(i), orig_sel_alias))) { LOG_WARN("failed to get column_name from origin select_items", K(ret)); - } else if (OB_FAIL(create_simple_column_expr(mlog_table->get_table_name(), rowkey_column->get_column_name_str(), mlog_table->table_id_, inner_expr)) + } else if (OB_FAIL(create_simple_column_expr(delta_src_table->get_table_name(), rowkey_column->get_column_name_str(), delta_src_table->table_id_, inner_expr)) || OB_FAIL(create_simple_column_expr(outer_table->get_table_name(), use_orig_sel_alias ? *orig_sel_alias : rowkey_column->get_column_name_str(), outer_table->table_id_, outer_expr))) { LOG_WARN("failed to create simple column expr", K(ret)); } else if (OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_EQ, outer_expr, inner_expr, filter))) { @@ -2291,11 +2317,11 @@ int ObMVPrinter::gen_update_insert_delete_for_simple_join_mav(ObIArray &dml_stmts) +int ObMVPrinter::gen_merge_for_simple_join_mav(ObIArray &dml_stmts) { int ret = OB_SUCCESS; dml_stmts.reuse(); - ObSEArray inner_delta_mavs; + ObSEArray inner_delta_mavs; if (OB_FAIL(gen_inner_delta_mav_for_simple_join_mav(inner_delta_mavs))) { LOG_WARN("failed to gen inner delta mav for simple join mav", K(ret)); } else { @@ -2454,8 +2480,7 @@ int ObMVPrinter::gen_delta_data_access_stmt(const TableItem &source_table, // from (select ... from mlog_t where ora_rowscn > xxx and ora_rowscn < xxx) // where (old_new = 'O' and seq_no = "MINSEQ$$") // ; -int ObMVPrinter::gen_pre_data_access_stmt(const TableItem &source_table, - ObSelectStmt *&access_sel) +int ObMVPrinter::gen_pre_data_access_stmt(const TableItem &source_table, ObSelectStmt *&access_sel) { int ret = OB_SUCCESS; ObSelectStmt *union_stmt = NULL; @@ -2467,8 +2492,8 @@ int ObMVPrinter::gen_pre_data_access_stmt(const TableItem &source_table, LOG_WARN("failed to unchanged deleted data access stmt ", K(ret)); } else if (OB_FAIL(gen_deleted_data_access_stmt(source_table, deleted_data_stmt))) { LOG_WARN("failed to generate deleted data access stmt ", K(ret)); - } else if (OB_FAIL(union_stmt->get_set_query().push_back(unchanged_data_stmt) - || OB_FAIL(union_stmt->get_set_query().push_back(deleted_data_stmt)))) { + } else if (OB_FAIL(union_stmt->get_set_query().push_back(unchanged_data_stmt) || + OB_FAIL(union_stmt->get_set_query().push_back(deleted_data_stmt)))) { LOG_WARN("failed to set set query", K(ret)); } else { union_stmt->assign_set_all(); @@ -2524,5 +2549,531 @@ int ObMVPrinter::gen_deleted_data_access_stmt(const TableItem &source_table, return ret; } + +int ObMVPrinter::gen_refresh_select_for_major_refresh_mjv(ObIArray &dml_stmts) +{ + int ret = OB_SUCCESS; + dml_stmts.reuse(); + ObSelectStmt *left_delta_stmt = NULL; + ObSelectStmt *right_delta_stmt = NULL; + ObSEArray rowkey_sel_pos; + if (OB_FAIL(get_rowkey_pos_in_select(rowkey_sel_pos))) { + LOG_WARN("failed to get rowkey pos in select", K(ret)); + } else if (OB_FAIL(gen_one_refresh_select_for_major_refresh_mjv(rowkey_sel_pos, true, left_delta_stmt))) { + LOG_WARN("failed to generate gen left refresh select for major refresh mjv ", K(ret)); + } else if (OB_FAIL(gen_one_refresh_select_for_major_refresh_mjv(rowkey_sel_pos, false, right_delta_stmt))) { + LOG_WARN("failed to generate gen right refresh select for major refresh mjv ", K(ret)); + } else if (OB_FAIL(dml_stmts.push_back(left_delta_stmt)) + || OB_FAIL(dml_stmts.push_back(right_delta_stmt))) { + LOG_WARN("failed to push back", K(ret)); + } + return ret; +} + +int ObMVPrinter::get_rowkey_pos_in_select(ObIArray &rowkey_sel_pos) +{ + int ret = OB_SUCCESS; + rowkey_sel_pos.reuse(); + const ObTableSchema *container_table = NULL; + if (OB_ISNULL(stmt_factory_.get_query_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(stmt_factory_.get_query_ctx())); + } else if (OB_FAIL(stmt_factory_.get_query_ctx()->sql_schema_guard_.get_table_schema(mv_schema_.get_data_table_id(), container_table))) { + LOG_WARN("failed to get table schema", K(ret)); + } else if (OB_ISNULL(container_table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(container_table)); + } else { + const ObRowkeyInfo &rowkey_info = container_table->get_rowkey_info(); + const ObRowkeyColumn *rowkey_column = NULL; + const int64_t sel_count = mv_checker_.get_stmt().get_select_items().count(); + int64_t pos = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_info.get_size(); ++i) { + if (OB_ISNULL(rowkey_column = rowkey_info.get_column(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("The rowkey column is NULL, ", K(i)); + } else if (OB_UNLIKELY(0 > (pos = rowkey_column->column_id_ - OB_APP_MIN_COLUMN_ID) || sel_count <= pos)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected rowkey column", K(pos), K(rowkey_column->column_id_)); + } else if (OB_FAIL(rowkey_sel_pos.push_back(pos))) { + LOG_WARN("failed to push back", K(ret)); + } + } + LOG_WARN("finish get rowkey pos in select", K(rowkey_sel_pos)); + } + return ret; +} + +/* + select * from mv where + not exists (select 1 from t1 where t1.pk = mv.t1_pk and $$old_new = 'O' and ora_rowscn > last_refresh_scn(mv_id)) + and not exists (select 1 from t2 where t2.pk = mv.t2_pk and $$old_new = 'O' and ora_rowscn > last_refresh_scn(mv_id)) + union all + select * from t1, t2 + where t1.c1 = t2.c1 + and t2.ora_rowscn <= last_refresh_scn(mv_id) + and t1.ora_rowscn > last_refresh_scn(mv_id) + and t1.$$old_new = 'N' + union all + select * from t1, t2 where t1.c1 = t2.c1 and t2.ora_rowscn > last_refresh_scn(mv_id) and t2.$$old_new = 'N'; +*/ +int ObMVPrinter::gen_real_time_view_for_major_refresh_mjv(ObSelectStmt *&sel_stmt) +{ + int ret = OB_SUCCESS; + sel_stmt = NULL; + ObSelectStmt *access_mv_stmt = NULL; + ObSelectStmt *delta_left_stmt = NULL; + ObSelectStmt *delta_right_stmt = NULL; + if (OB_FAIL(create_simple_stmt(sel_stmt))) { + LOG_WARN("failed to create simple stmt", K(ret)); + } else if (OB_FAIL(gen_access_mv_data_for_major_refresh_mjv(access_mv_stmt))) { + LOG_WARN("failed to generate access mv data for major refresh mjv", K(ret)); + } else if (OB_FAIL(gen_real_time_view_access_delta_data_for_major_refresh_mjv(delta_left_stmt, delta_right_stmt))) { + LOG_WARN("failed to generate real time view access delta data for major refresh mjv", K(ret)); + } else if (OB_FAIL(sel_stmt->get_set_query().push_back(access_mv_stmt)) + || OB_FAIL(sel_stmt->get_set_query().push_back(delta_left_stmt)) + || OB_FAIL(sel_stmt->get_set_query().push_back(delta_right_stmt))) { + LOG_WARN("failed to set set query", K(ret)); + } else { + sel_stmt->assign_set_all(); + sel_stmt->assign_set_op(ObSelectStmt::UNION); + } + return ret; +} + +int ObMVPrinter::fill_table_partition_name(const TableItem &src_table, + TableItem &table) +{ + int ret = OB_SUCCESS; + const ObTableSchema *src_schema = NULL; + table.part_names_.reuse(); + table.part_ids_.reuse(); + ObPartition *part = NULL; + ObSubPartition *subpart = NULL; + int64_t part_idx = NULL != major_refresh_info_ ? major_refresh_info_->part_idx_ : OB_INVALID_INDEX; + int64_t sub_part_idx = NULL != major_refresh_info_ ? major_refresh_info_->sub_part_idx_ : OB_INVALID_INDEX; + if (OB_ISNULL(stmt_factory_.get_query_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(stmt_factory_.get_query_ctx())); + } else if (OB_INVALID_INDEX == part_idx && OB_INVALID_INDEX == sub_part_idx) { + /* do nothing */ + } else if (OB_FAIL(stmt_factory_.get_query_ctx()->sql_schema_guard_.get_table_schema(src_table.ref_id_, src_schema))) { + LOG_WARN("failed to get table schema", K(ret)); + } else if (OB_ISNULL(src_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(src_schema)); + } else if (OB_UNLIKELY(PARTITION_LEVEL_ONE != src_schema->get_part_level() + && PARTITION_LEVEL_TWO != src_schema->get_part_level())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected table", K(ret), K(part_idx), K(sub_part_idx), KPC(src_schema)); + } else if (OB_UNLIKELY(part_idx < 0 || part_idx >= src_schema->get_partition_capacity()) + || OB_ISNULL(part = src_schema->get_part_array()[part_idx])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected partition", K(ret), K(part_idx), K(part), KPC(src_schema)); + } else if (OB_FAIL(table.part_ids_.push_back(part_idx))) { // just push a invalid part id to print partition name + LOG_WARN("failed to push back", K(ret)); + } else if (PARTITION_LEVEL_ONE == src_schema->get_part_level()) { + if (OB_FAIL(table.part_names_.push_back(part->get_part_name()))) { + LOG_WARN("failed to push back", K(ret)); + } + } else if (OB_UNLIKELY(sub_part_idx < 0 || sub_part_idx >= part->get_subpartition_num()) + || OB_ISNULL(subpart = part->get_subpart_array()[sub_part_idx])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected subpartition", K(ret), K(sub_part_idx), KPC(part), K(subpart), KPC(src_schema)); + } else if (OB_FAIL(table.part_names_.push_back(subpart->get_part_name()))) { + LOG_WARN("failed to push back", K(ret)); + } + return ret; +} + +int ObMVPrinter::append_rowkey_range_filter(const ObIArray &select_items, + uint64_t rowkey_count, + ObIArray &conds) +{ + int ret = OB_SUCCESS; + const ObNewRange *range = NULL != major_refresh_info_ ? &major_refresh_info_->range_ : NULL; + if (NULL == range) { + /* do nothing */ + } else if (OB_UNLIKELY(select_items.count() < rowkey_count + || rowkey_count < range->start_key_.get_obj_cnt() + || rowkey_count < range->end_key_.get_obj_cnt())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected params", K(ret), K(select_items.count()), KPC(range)); + } else { + ObRawExpr *filter = NULL; + ObRawExpr *col = NULL; + uint64_t s_cnt = range->start_key_.get_obj_cnt(); + uint64_t e_cnt = range->end_key_.get_obj_cnt(); + const ObObj *s_obj = range->start_key_.get_obj_ptr(); + const ObObj *e_obj = range->end_key_.get_obj_ptr(); + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_count; ++i) { + const ObObj *obj1 = i < s_cnt ? s_obj + i : NULL; + const ObObj *obj2 = i < e_cnt ? e_obj + i : NULL; + ObConstRawExpr *const_expr1 = NULL; + ObConstRawExpr *const_expr2 = NULL; + bool euqal = NULL != obj1 && NULL != obj2 && obj1->is_equal(*obj2); + ObItemType cmp_type1 = euqal ? T_OP_EQ : (range->border_flag_.inclusive_start() ? T_OP_GE : T_OP_GT); + ObItemType cmp_type2 = range->border_flag_.inclusive_end() ? T_OP_LE : T_OP_LT; + if (OB_ISNULL(col = select_items.at(i).expr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(i), K(col)); + } else if (NULL != obj1 && !obj1->is_min_value() && + OB_FAIL(ObRawExprUtils::build_const_obj_expr(expr_factory_, *obj1, const_expr1))) { + LOG_WARN("failed to build const obj expr", K(ret)); + } else if (NULL != obj2 && !obj2->is_max_value() && !euqal && + OB_FAIL(ObRawExprUtils::build_const_obj_expr(expr_factory_, *obj2, const_expr2))) { + LOG_WARN("failed to build const obj expr", K(ret)); + } else if (NULL != const_expr1 && + (OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, cmp_type1, col, const_expr1, filter)) + || OB_FAIL(conds.push_back(filter)))) { + LOG_WARN("failed to build and push back binary op expr", K(ret)); + } else if (NULL != const_expr2 && + (OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, cmp_type2, col, const_expr2, filter)) + || OB_FAIL(conds.push_back(filter)))) { + LOG_WARN("failed to build and push back binary op expr", K(ret)); + } + } + } + return ret; +} + +int ObMVPrinter::gen_access_mv_data_for_major_refresh_mjv(ObSelectStmt *&sel_stmt) +{ + int ret = OB_SUCCESS; + sel_stmt = NULL; + TableItem *mv_table = NULL; + const ObIArray &orig_select_items = mv_checker_.get_stmt().get_select_items(); + const ObIArray &orig_table_items = mv_checker_.get_stmt().get_table_items(); + if (OB_FAIL(create_simple_stmt(sel_stmt))) { + LOG_WARN("failed to create simple stmt", K(ret)); + } else if (OB_FAIL(create_simple_table_item(sel_stmt, mv_schema_.get_table_name(), mv_table))) { + LOG_WARN("failed to create simple table item", K(ret)); + } else if (OB_FAIL(sel_stmt->get_select_items().prepare_allocate(orig_select_items.count())) + || OB_FAIL(sel_stmt->get_condition_exprs().prepare_allocate(orig_table_items.count()))) { + LOG_WARN("failed to prepare allocate arrays", K(ret)); + } else { + mv_table->database_name_ = mv_db_name_; + mv_table->flashback_query_expr_ = exprs_.last_refresh_scn_; + mv_table->flashback_query_type_ = TableItem::USING_SCN; + ObIArray &select_items = sel_stmt->get_select_items(); + ObIArray &conds = sel_stmt->get_condition_exprs(); + for (int64_t i = 0; OB_SUCC(ret) && i < select_items.count(); ++i) { + if (OB_FAIL(create_simple_column_expr(mv_table->get_table_name(), orig_select_items.at(i).alias_name_, + mv_table->table_id_, select_items.at(i).expr_))) { + LOG_WARN("failed to create simple column exprs", K(ret)); + } else { + select_items.at(i).is_real_alias_ = true; + select_items.at(i).alias_name_ = orig_select_items.at(i).alias_name_; + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < orig_table_items.count(); ++i) { + if (OB_FAIL(gen_exists_cond_for_table(orig_table_items.at(i), mv_table, false, true, conds.at(i), false))) { + LOG_WARN("failed to create simple column exprs", K(ret)); + } + } + } + return ret; +} + +/* + generate stmt: + 1. delat_t1 join pre_t2 + select * from t1, t2 + where t1.c1 = t2.c1 + and t2.ora_rowscn <= last_refresh_scn(mv_id) + and t1.ora_rowscn > last_refresh_scn(mv_id) + and t1.$$old_new = 'N' + 2. t1 join delta_t2 + select * from t1, t2 where t1.c1 = t2.c1 and t2.ora_rowscn > last_refresh_scn(mv_id) and t2.$$old_new = 'N'; +*/ +int ObMVPrinter::gen_real_time_view_access_delta_data_for_major_refresh_mjv(ObSelectStmt *&delta_left_stmt, + ObSelectStmt *&delta_right_stmt) +{ + int ret = OB_SUCCESS; + delta_left_stmt = NULL; + delta_right_stmt = NULL; + ObSelectStmt *src_sel_stmt = NULL; + const ObIArray &orig_table_items = mv_checker_.get_stmt().get_table_items(); + const TableItem *left_table = NULL; + const TableItem *right_table = NULL; + ObRawExpr *left_col = NULL; + ObRawExpr *right_col = NULL; + ObRawExpr *left_scn_gt = NULL; + ObRawExpr *right_scn_le = NULL; + ObRawExpr *right_scn_gt = NULL; + ObConstRawExpr *const_expr = exprs_.str_n_; + ObSEArray dummy_array; + if (OB_UNLIKELY(2 != orig_table_items.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected params", K(ret), K(orig_table_items)); + } else if (OB_FAIL(create_simple_stmt(src_sel_stmt))) { + LOG_WARN("failed to create simple stmt", K(ret)); + } else if (OB_FAIL(prepare_gen_access_delta_data_for_major_refresh_mjv(dummy_array, *src_sel_stmt))) { + LOG_WARN("failed to prepare generate access delta data for major refresh mjv", K(ret)); + } else if (OB_UNLIKELY(2 != src_sel_stmt->get_table_items().count()) + || OB_ISNULL(left_table = src_sel_stmt->get_table_item(0)) + || OB_ISNULL(right_table = src_sel_stmt->get_table_item(1))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected params", K(ret), K(orig_table_items)); + } else if (OB_FAIL(create_simple_column_expr(left_table->get_table_name(), ObString("ORA_ROWSCN"), left_table->table_id_, left_col)) + || OB_FAIL(create_simple_column_expr(right_table->get_table_name(), ObString("ORA_ROWSCN"), right_table->table_id_, right_col))) { + LOG_WARN("failed to create simple column expr", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_GT, left_col, exprs_.last_refresh_scn_, left_scn_gt)) + || OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_LE, right_col, exprs_.last_refresh_scn_, right_scn_le)) + || OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_GT, right_col, exprs_.last_refresh_scn_, right_scn_gt))) { + LOG_WARN("failed to build greater op expr", K(ret)); + } else if (OB_FAIL(create_simple_stmt(delta_left_stmt)) + || OB_FAIL(assign_simple_sel_stmt(*delta_left_stmt, *src_sel_stmt)) + || OB_FAIL(delta_left_stmt->get_condition_exprs().push_back(left_scn_gt)) + || OB_FAIL(delta_left_stmt->get_condition_exprs().push_back(right_scn_le)) + || OB_FAIL(append_old_new_col_filter(*left_table, const_expr, delta_left_stmt->get_condition_exprs()))) { + LOG_WARN("failed to create left delta table select stmt", K(ret)); + } else if (OB_FAIL(src_sel_stmt->get_condition_exprs().push_back(right_scn_gt)) + || OB_FAIL(append_old_new_col_filter(*right_table, const_expr, src_sel_stmt->get_condition_exprs()))) { + LOG_WARN("failed to append old new col filter", K(ret)); + } else { + delta_right_stmt = src_sel_stmt; + } + return ret; +} + +/* + generate stmt: + 1. delat_t1 join pre_t2 + select pk1, pk2, ..., t1.$$old_new from t1 as of snapshot current_scn, t2 as of snapshot last_scn + where t1.c1 = t2.c1 + and t1.ora_rowscn > last_scn + and t1.$$old_new = 'F' + order by 1,2; + 2. t1 join delta_t2 + select pk1, pk2, ..., t2.$$old_new from t1 as of snapshot current_scn, t2 as of snapshot current_scn + where t1.c1 = t2.c1 + and t2.ora_rowscn > last_scn + and t2.$$old_new = 'F' + order by 1,2; +*/ +int ObMVPrinter::gen_one_refresh_select_for_major_refresh_mjv(const ObIArray &rowkey_sel_pos, + const bool is_delta_left, + ObSelectStmt *&delta_stmt) +{ + int ret = OB_SUCCESS; + delta_stmt = NULL; + const ObIArray &orig_table_items = mv_checker_.get_stmt().get_table_items(); + const TableItem *orig_left_table = NULL; + const TableItem *delta_table = NULL; + TableItem *left_table = NULL; + TableItem *right_table = NULL; + ObRawExpr *col = NULL; + ObRawExpr *scn_filter = NULL; + ObConstRawExpr *const_expr = NULL; + SelectItem sel_item; + sel_item.is_real_alias_ = true; + sel_item.alias_name_ = OLD_NEW_COL_NAME; + if (OB_UNLIKELY(2 != orig_table_items.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected params", K(ret), K(orig_table_items)); + } else if (OB_FAIL(create_simple_stmt(delta_stmt)) + || OB_FAIL(prepare_gen_access_delta_data_for_major_refresh_mjv(rowkey_sel_pos, *delta_stmt))) { + LOG_WARN("failed to prepare generate access delta data for major refresh mjv", K(ret)); + } else if (OB_UNLIKELY(2 != delta_stmt->get_table_items().count() || 2 != mv_checker_.get_stmt().get_table_items().count()) + || OB_ISNULL(orig_left_table = mv_checker_.get_stmt().get_table_item(0)) + || OB_ISNULL(left_table = delta_stmt->get_table_item(0)) + || OB_ISNULL(right_table = delta_stmt->get_table_item(1))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected params", K(ret)); + } else if (OB_FALSE_IT(delta_table = is_delta_left ? left_table : right_table)) { + } else if (OB_FAIL(ObRawExprUtils::build_const_string_expr(expr_factory_, ObVarcharType, ObString("F"), + ObCharset::get_default_collation(ObCharset::get_default_charset()), + const_expr))) { + LOG_WARN("fail to build const string expr", K(ret)); + } else if (OB_FAIL(create_simple_column_expr(delta_table->get_table_name(), ObString("ORA_ROWSCN"), delta_table->table_id_, col)) + || OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_GT, col, exprs_.last_refresh_scn_, scn_filter)) + || OB_FAIL(delta_stmt->get_condition_exprs().push_back(scn_filter)) + || OB_FAIL(append_old_new_col_filter(*delta_table, const_expr, delta_stmt->get_condition_exprs()))) { + LOG_WARN("failed to add filter for delta left stmt", K(ret)); + } else if (OB_FAIL(fill_table_partition_name(*orig_left_table, *left_table))) { + LOG_WARN("failed to fill table partition name", K(ret)); + } else if (OB_FAIL(create_simple_column_expr(delta_table->get_table_name(), OLD_NEW_COL_NAME, delta_table->table_id_, sel_item.expr_)) + || OB_FAIL(delta_stmt->get_select_items().push_back(sel_item))) { + LOG_WARN("failed to add old_new to select items", K(ret)); + } else if (OB_FAIL(gen_refresh_select_hint_for_major_refresh_mjv(*left_table, *right_table, delta_stmt->get_stmt_hint()))) { + LOG_WARN("failed to gen refresh select hint for major refresh mjv", K(ret)); + } else { + left_table->flashback_query_expr_ = exprs_.refresh_scn_; + left_table->flashback_query_type_ = TableItem::USING_SCN; + right_table->flashback_query_expr_ = is_delta_left ? exprs_.last_refresh_scn_ : exprs_.refresh_scn_; + right_table->flashback_query_type_ = TableItem::USING_SCN; + } + return ret; +} + +int ObMVPrinter::gen_refresh_select_hint_for_major_refresh_mjv(const TableItem &left_table, + const TableItem &right_table, + ObStmtHint &stmt_hint) +{ + int ret = OB_SUCCESS; + ObJoinOrderHint *leading_left = NULL; + ObJoinHint *use_nl_right = NULL; + ObIndexHint *left_full_hint = NULL; + ObIndexHint *right_full_hint = NULL; + ObSEArray conflict_hints; + if (OB_FAIL(ObQueryHint::create_hint(&alloc_, T_LEADING, leading_left)) || + OB_FAIL(ObQueryHint::create_hint(&alloc_, T_FULL_HINT, left_full_hint)) || + OB_FAIL(ObQueryHint::create_hint(&alloc_, T_FULL_HINT, right_full_hint)) || + OB_FAIL(ObQueryHint::create_hint(&alloc_, T_USE_NL, use_nl_right))) { + LOG_WARN("failed to create hint", K(ret)); + } else if (OB_FAIL(ObQueryHint::create_hint_table(&alloc_, leading_left->get_table().table_))) { + LOG_WARN("fail to create hint table", K(ret)); + } else if (OB_FALSE_IT(leading_left->get_table().table_->set_table(left_table))) { + } else if (OB_FALSE_IT(left_full_hint->get_table().set_table(left_table))) { + } else if (OB_FALSE_IT(right_full_hint->get_table().set_table(right_table))) { + } else if (OB_FAIL(use_nl_right->get_tables().push_back(ObTableInHint(right_table)))) { + LOG_WARN("fail to push back", K(ret)); + } else if (OB_FAIL(stmt_hint.merge_hint(*leading_left, ObHintMergePolicy::HINT_DOMINATED_EQUAL, conflict_hints)) + || OB_FAIL(stmt_hint.merge_hint(*left_full_hint, ObHintMergePolicy::HINT_DOMINATED_EQUAL, conflict_hints)) + || OB_FAIL(stmt_hint.merge_hint(*right_full_hint, ObHintMergePolicy::HINT_DOMINATED_EQUAL, conflict_hints)) + || OB_FAIL(stmt_hint.merge_hint(*use_nl_right, ObHintMergePolicy::HINT_DOMINATED_EQUAL, conflict_hints))) { + LOG_WARN("failed to merge hint", K(ret)); + } + return ret; +} + +int ObMVPrinter::assign_simple_sel_stmt(ObSelectStmt &target_stmt, ObSelectStmt &source_stmt) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(target_stmt.get_joined_tables().assign(source_stmt.get_joined_tables())) + || OB_FAIL(target_stmt.get_table_items().assign(source_stmt.get_table_items())) + || OB_FAIL(target_stmt.get_from_items().assign(source_stmt.get_from_items())) + || OB_FAIL(target_stmt.get_select_items().assign(source_stmt.get_select_items())) + || OB_FAIL(target_stmt.get_condition_exprs().assign(source_stmt.get_condition_exprs())) + || OB_FAIL(target_stmt.get_order_items().assign(source_stmt.get_order_items()))) { + LOG_WARN("failed to assign structure", K(ret)); + } + return ret; +} + +int ObMVPrinter::prepare_gen_access_delta_data_for_major_refresh_mjv(const ObIArray &rowkey_sel_pos, + ObSelectStmt &base_delta_stmt) +{ + int ret = OB_SUCCESS; + const ObIArray &orig_select_items = mv_checker_.get_stmt().get_select_items(); + const ObIArray &orig_table_items = mv_checker_.get_stmt().get_table_items(); + ObIArray &select_items = base_delta_stmt.get_select_items(); + ObIArray &order_items = base_delta_stmt.get_order_items(); + ObSEArray column_items; + ObRawExpr *expr = NULL; + const TableItem *orig_table = NULL; + TableItem *table = NULL; + ObRawExprCopier copier(expr_factory_); + int64_t idx = OB_INVALID_INDEX; + if (OB_UNLIKELY(!for_rt_expand_ && rowkey_sel_pos.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected params", K(ret), K(rowkey_sel_pos)); + } else if (OB_FAIL(select_items.prepare_allocate(orig_select_items.count())) + || OB_FAIL(order_items.prepare_allocate(rowkey_sel_pos.count()))) { + LOG_WARN("failed to prepare allocate arrays", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < orig_table_items.count(); ++i) { + column_items.reuse(); + if (OB_ISNULL(orig_table = orig_table_items.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(i), K(orig_table_items)); + } else if (OB_FAIL(mv_checker_.get_stmt().get_column_items(orig_table->table_id_, column_items))) { + LOG_WARN("failed to get column items", K(ret)); + } else if (OB_FAIL(create_simple_table_item(&base_delta_stmt, orig_table->table_name_, table))) { + LOG_WARN("failed to create simple table item", K(ret)); + } else { + table->alias_name_ = orig_table->alias_name_; + table->synonym_name_ = orig_table->synonym_name_; + table->database_name_ = orig_table->database_name_; + table->synonym_db_name_ = orig_table->synonym_db_name_; + } + for (int64_t j = 0; OB_SUCC(ret) && j < column_items.count(); ++j) { + if (OB_FAIL(create_simple_column_expr(table->get_table_name(), column_items.at(j).column_name_, + table->table_id_, expr))) { + LOG_WARN("failed to create simple column expr", K(ret)); + } else if (OB_FAIL(copier.add_replaced_expr(column_items.at(j).expr_, expr))) { + LOG_WARN("failed to add replace pair", K(ret)); + } + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(base_delta_stmt.deep_copy_join_tables(alloc_, copier, mv_checker_.get_stmt()))) { + LOG_WARN("failed to deep copy join tables", K(ret)); + } else if (OB_FAIL(copier.copy_on_replace(mv_checker_.get_stmt().get_condition_exprs(), base_delta_stmt.get_condition_exprs()))) { + LOG_WARN("failed to deep copy where conditions", K(ret)); + } else if (OB_FAIL(base_delta_stmt.get_from_items().assign(mv_checker_.get_stmt().get_from_items()))) { + LOG_WARN("failed to assign from items", K(ret)); + } + + // for non joined table, adjust table id in from item + ObIArray &from_items = base_delta_stmt.get_from_items(); + for (int64_t i = 0; OB_SUCC(ret) && i < from_items.count(); ++i) { + if (from_items.at(i).is_joined_) { + /* do nothing */ + } else if (OB_FAIL(mv_checker_.get_stmt().get_table_item_idx(from_items.at(i).table_id_, idx))) { + LOG_WARN("failed to get table item", K(ret)); + } else if (OB_UNLIKELY(idx < 0 || idx >= base_delta_stmt.get_table_size()) + || OB_ISNULL(base_delta_stmt.get_table_item(idx))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected idx", K(ret), K(idx), K(base_delta_stmt.get_table_size())); + } else { + from_items.at(i).table_id_ = base_delta_stmt.get_table_item(idx)->table_id_; + } + } + if (OB_FAIL(ret)) { + } else if (for_rt_expand_) { // for rt-mview access, generate original select list + for (int64_t i = 0; OB_SUCC(ret) && i < orig_select_items.count(); ++i) { + if (OB_FAIL(copier.copy_on_replace(orig_select_items.at(i).expr_, select_items.at(i).expr_))) { + LOG_WARN("failed to generate group by exprs", K(ret)); + } else { + select_items.at(i).is_real_alias_ = true; + select_items.at(i).alias_name_ = orig_select_items.at(i).alias_name_; + } + } + } else { // for refresh, generate select list as pk and other column, add order by + int64_t idx = OB_INVALID_INDEX; + int64_t pos = rowkey_sel_pos.count(); + for (int64_t i = 0; OB_SUCC(ret) && i < orig_select_items.count(); ++i) { + if (OB_FAIL(copier.copy_on_replace(orig_select_items.at(i).expr_, expr))) { + LOG_WARN("failed to generate group by exprs", K(ret)); + } else if (ObOptimizerUtil::find_item(rowkey_sel_pos, i, &idx)) { + order_items.at(idx).expr_ = expr; + select_items.at(idx).expr_ = expr; + select_items.at(idx).alias_name_ = orig_select_items.at(i).alias_name_; + select_items.at(idx).is_real_alias_ = true; + } else { + select_items.at(pos).expr_ = expr; + select_items.at(pos).alias_name_ = orig_select_items.at(i).alias_name_; + select_items.at(pos).is_real_alias_ = true; + ++pos; + } + } + if (OB_SUCC(ret) && OB_FAIL(append_rowkey_range_filter(select_items, + rowkey_sel_pos.count(), + base_delta_stmt.get_condition_exprs()))) { + LOG_WARN("failed to append rowkey range filter", K(ret)); + } + } + return ret; +} + +int ObMVPrinter::append_old_new_col_filter(const TableItem &table, + ObRawExpr *val, + ObIArray& conds) +{ + int ret = OB_SUCCESS; + ObRawExpr *col = NULL; + ObRawExpr *filter = NULL; + if (OB_FAIL(create_simple_column_expr(table.get_table_name(), OLD_NEW_COL_NAME, table.table_id_, col))) { + LOG_WARN("failed to create simple column expr", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, T_OP_EQ, col, val, filter))) { + LOG_WARN("failed to build mul expr", K(ret)); + } else if (OB_FAIL(conds.push_back(filter))) { + LOG_WARN("failed to push back expr", K(ret)); + } + return ret; +} + }//end of namespace sql }//end of namespace oceanbase diff --git a/src/sql/resolver/mv/ob_mv_printer.h b/src/sql/resolver/mv/ob_mv_printer.h index 2de91012e1..dbb2e365ac 100644 --- a/src/sql/resolver/mv/ob_mv_printer.h +++ b/src/sql/resolver/mv/ob_mv_printer.h @@ -50,6 +50,22 @@ struct SharedPrinterRawExprs ObConstRawExpr *str_o_; }; +struct MajorRefreshInfo { + MajorRefreshInfo(const int64_t part_idx, const int64_t sub_part_idx, const ObNewRange &range) + : part_idx_(part_idx), + sub_part_idx_(sub_part_idx), + range_(range) + {} + + bool is_valid_info() const { return OB_INVALID_INDEX != part_idx_ || OB_INVALID_INDEX != sub_part_idx_ || !range_.is_whole_range();} + + TO_STRING_KV(K_(part_idx), K_(sub_part_idx), K_(range)); + const int64_t part_idx_; + const int64_t sub_part_idx_; + const ObNewRange &range_; + DISALLOW_COPY_AND_ASSIGN(MajorRefreshInfo); +}; + class ObMVPrinter { public: @@ -66,7 +82,8 @@ public: for_rt_expand_(for_rt_expand), stmt_factory_(stmt_factory), expr_factory_(expr_factory), - exprs_() + exprs_(), + major_refresh_info_(NULL) {} ~ObMVPrinter() {} @@ -84,10 +101,12 @@ public: static const ObString WIN_MIN_SEQ_COL_NAME; static int print_mv_operators(const share::schema::ObTableSchema &mv_schema, + const share::schema::ObTableSchema &mv_container_schema, const ObSelectStmt &view_stmt, const bool for_rt_expand, const share::SCN &last_refresh_scn, const share::SCN &refresh_scn, + const MajorRefreshInfo *major_refresh_info, ObIAllocator &alloc, ObIAllocator &str_alloc, ObSchemaGetterGuard *schema_guard, @@ -98,7 +117,6 @@ public: ObMVRefreshableType &refreshable_type); private: - enum MlogExtColFlag { MLOG_EXT_COL_OLD_NEW = 0x1, MLOG_EXT_COL_SEQ = 0x1 << 1, @@ -107,7 +125,7 @@ private: MLOG_EXT_COL_WIN_MAX_SEQ = 0x1 << 4, }; - int init(const share::SCN &last_refresh_scn, const share::SCN &refresh_scn); + int init(const share::SCN &last_refresh_scn, const share::SCN &refresh_scn, const MajorRefreshInfo *major_refresh_info); int gen_mv_operator_stmts(ObIArray &dml_stmts); int gen_refresh_dmls_for_mv(ObIArray &dml_stmts); int gen_real_time_view_for_mv(ObSelectStmt *&sel_stmt); @@ -135,7 +153,8 @@ private: const TableItem *outer_table, const bool is_exists, const bool use_orig_sel_alias, - ObRawExpr *&exists_expr); + ObRawExpr *&exists_expr, + bool use_mlog = true); int get_column_name_from_origin_select_items(const uint64_t table_id, const uint64_t column_id, const ObString *&col_name); @@ -256,6 +275,30 @@ private: TableItem *&table_item, ObSelectStmt *view_stmt = NULL, const bool add_to_from = true); + + int gen_refresh_select_for_major_refresh_mjv(ObIArray &dml_stmts); + int get_rowkey_pos_in_select(ObIArray &rowkey_sel_pos); + int gen_real_time_view_for_major_refresh_mjv(ObSelectStmt *&sel_stmt); + int gen_access_mv_data_for_major_refresh_mjv(ObSelectStmt *&sel_stmt); + int gen_not_exists_cond_for_major_refresh_mjv(const ObIArray &upper_sel_exprs, + const TableItem *source_table, + ObRawExpr *&exists_expr); + int gen_real_time_view_access_delta_data_for_major_refresh_mjv(ObSelectStmt *&delta_left_stmt, + ObSelectStmt *&delta_right_stmt); + int gen_one_refresh_select_for_major_refresh_mjv(const ObIArray &rowkey_sel_pos, + const bool is_delta_left, + ObSelectStmt *&delta_stmt); + int gen_refresh_select_hint_for_major_refresh_mjv(const TableItem &left_table, + const TableItem &right_table, + ObStmtHint &stmt_hint); + int prepare_gen_access_delta_data_for_major_refresh_mjv(const ObIArray &rowkey_sel_pos, + ObSelectStmt &base_delta_stmt); + int append_old_new_col_filter(const TableItem &table, ObRawExpr *val, ObIArray& conds); + int fill_table_partition_name(const TableItem &src_table, TableItem &table); + int append_rowkey_range_filter(const ObIArray &select_items, + uint64_t rowkey_count, + ObIArray &conds); + int assign_simple_sel_stmt(ObSelectStmt &target_stmt, ObSelectStmt &source_stmt); template inline int create_simple_stmt(StmtType *&stmt) { @@ -288,6 +331,7 @@ private: ObStmtFactory &stmt_factory_; ObRawExprFactory &expr_factory_; SharedPrinterRawExprs exprs_; + const MajorRefreshInfo *major_refresh_info_; DISALLOW_COPY_AND_ASSIGN(ObMVPrinter); }; diff --git a/src/sql/resolver/mv/ob_mv_provider.cpp b/src/sql/resolver/mv/ob_mv_provider.cpp index bed0baf930..7bd7ed6e94 100644 --- a/src/sql/resolver/mv/ob_mv_provider.cpp +++ b/src/sql/resolver/mv/ob_mv_provider.cpp @@ -27,6 +27,27 @@ using namespace common; namespace sql { +// 1. resolve mv definition and get stmt +// 2. check refresh type by stmt +// 3. print refresh dmls +int ObMVProvider::init_mv_provider(const share::SCN &last_refresh_scn, + const share::SCN &refresh_scn, + ObSchemaGetterGuard *schema_guard, + ObSQLSessionInfo *session_info, + int64_t part_idx, + int64_t sub_part_idx, + ObNewRange &range) +{ + int ret = OB_SUCCESS; + MajorRefreshInfo major_refresh_info(part_idx, sub_part_idx, range); + major_refresh_info_ = major_refresh_info.is_valid_info() ? &major_refresh_info : NULL; + if (OB_FAIL(init_mv_provider(last_refresh_scn, refresh_scn, schema_guard, session_info))) { + LOG_WARN("Failed to init mv provider", K(ret), K(major_refresh_info)); + } + major_refresh_info_ = NULL; + return ret; +} + // 1. resolve mv definition and get stmt // 2. check refresh type by stmt // 3. print refresh dmls @@ -56,6 +77,7 @@ int ObMVProvider::init_mv_provider(const share::SCN &last_refresh_scn, ObRawExprFactory expr_factory(alloc); ObSchemaChecker schema_checker; const ObTableSchema *mv_schema = NULL; + const ObTableSchema *mv_container_schema = NULL; ObQueryCtx *query_ctx = NULL; int64_t max_version = OB_INVALID_VERSION; ObSEArray operators; @@ -74,6 +96,7 @@ int ObMVProvider::init_mv_provider(const share::SCN &last_refresh_scn, *session_info, mview_id_, mv_schema, + mv_container_schema, view_stmt))) { LOG_WARN("failed to gen mv stmt", K(ret)); } else if (OB_FAIL(ObDependencyInfo::collect_dep_infos(query_ctx->reference_obj_tables_, @@ -93,8 +116,9 @@ int ObMVProvider::init_mv_provider(const share::SCN &last_refresh_scn, LOG_WARN("failed to check mv column type", K(ret)); } } else if (OB_FALSE_IT(query_ctx->get_query_hint_for_update().reset())) { // reset hint from mview definition - } else if (OB_FAIL(ObMVPrinter::print_mv_operators(*mv_schema, *view_stmt, for_rt_expand_, - last_refresh_scn, refresh_scn, + } else if (OB_FAIL(ObMVPrinter::print_mv_operators(*mv_schema, *mv_container_schema, + *view_stmt, for_rt_expand_, + last_refresh_scn, refresh_scn, major_refresh_info_, alloc, inner_alloc_, schema_guard, stmt_factory, @@ -285,11 +309,14 @@ int ObMVProvider::generate_mv_stmt(ObIAllocator &alloc, ObSQLSessionInfo &session_info, const uint64_t mv_id, const ObTableSchema *&mv_schema, + const ObTableSchema *&mv_container_schema, const ObSelectStmt *&view_stmt) { int ret = OB_SUCCESS; view_stmt = NULL; mv_schema = NULL; + mv_container_schema = NULL; + uint64_t mv_container_id = 0; ObString view_definition; ParseResult parse_result; ParseNode *node = NULL; @@ -314,6 +341,14 @@ int ObMVProvider::generate_mv_stmt(ObIAllocator &alloc, ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected mv schema", K(ret), KPC(mv_schema)); mv_schema = NULL; + } else if (FALSE_IT(mv_container_id = mv_schema->get_data_table_id())) { + + } else if (OB_FAIL(resolver_ctx.query_ctx_->sql_schema_guard_.get_table_schema(mv_container_id, + mv_container_schema))) { + LOG_WARN("fail to get mv container schema", KR(ret), K(mv_container_id)); + } else if (OB_ISNULL(mv_container_schema)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("unexpected null", KR(ret), K(mv_container_schema)); } else if (OB_FAIL(check_mview_dep_session_vars(*mv_schema, session_info, true, is_vars_matched))) { LOG_WARN("failed to check mview dep session vars", K(ret)); } else if (OB_FAIL(ObSQLUtils::generate_view_definition_for_resolve(alloc, @@ -323,13 +358,15 @@ int ObMVProvider::generate_mv_stmt(ObIAllocator &alloc, LOG_WARN("fail to generate view definition for resolve", K(ret)); } else if (OB_FAIL(parser.parse(view_definition, parse_result))) { LOG_WARN("parse view definition failed", K(view_definition), K(ret)); - } else if (OB_ISNULL(node = parse_result.result_tree_->children_[0]) || OB_UNLIKELY(T_SELECT != node->type_)) { + } else if (OB_ISNULL(node = parse_result.result_tree_->children_[0]) || + OB_UNLIKELY(T_SELECT != node->type_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid mv select node", K(ret), K(node), K(node->type_)); - } else if (OB_FALSE_IT(resolver_ctx.query_ctx_->question_marks_count_ = static_cast(parse_result.question_mark_ctx_.count_))) { + } else if (OB_FALSE_IT(resolver_ctx.query_ctx_->question_marks_count_ = + static_cast(parse_result.question_mark_ctx_.count_))) { } else if (OB_FAIL(select_resolver.resolve(*node))) { LOG_WARN("resolve view definition failed", K(ret)); - } else if (OB_ISNULL(sel_stmt = static_cast(select_resolver.get_basic_stmt()))) { + } else if (OB_ISNULL(sel_stmt = static_cast(select_resolver.get_basic_stmt()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid mv stmt", K(ret), K(sel_stmt)); } else if (OB_FAIL(sel_stmt->formalize_stmt_expr_reference(&expr_factory, &session_info, true))) { diff --git a/src/sql/resolver/mv/ob_mv_provider.h b/src/sql/resolver/mv/ob_mv_provider.h index b21a454dc2..97f198f604 100644 --- a/src/sql/resolver/mv/ob_mv_provider.h +++ b/src/sql/resolver/mv/ob_mv_provider.h @@ -19,6 +19,7 @@ #include "sql/resolver/expr/ob_raw_expr.h" #include "sql/resolver/mv/ob_mv_checker.h" #include "sql/resolver/dml/ob_merge_stmt.h" +#include "sql/resolver/mv/ob_mv_printer.h" namespace oceanbase { @@ -35,7 +36,8 @@ public: inited_(false), refreshable_type_(OB_MV_REFRESH_INVALID), operators_(&inner_alloc_), - dependency_infos_(&inner_alloc_) + dependency_infos_(&inner_alloc_), + major_refresh_info_(NULL) {} ~ObMVProvider() {} @@ -45,10 +47,19 @@ public: const share::SCN &refresh_scn, ObSchemaGetterGuard *schema_guard, ObSQLSessionInfo *session_info); + int init_mv_provider(const share::SCN &last_refresh_scn, + const share::SCN &refresh_scn, + ObSchemaGetterGuard *schema_guard, + ObSQLSessionInfo *session_info, + int64_t part_idx, + int64_t sub_part_idx, + ObNewRange &range); int check_mv_refreshable(bool &can_fast_refresh) const; int get_fast_refresh_operators(const ObIArray *&operators) const; int get_real_time_mv_expand_view(ObIAllocator &alloc, ObString &expand_view) const; int get_mv_dependency_infos(ObIArray &dep_infos) const; + OB_INLINE const common::ObIArray &get_operators() const + { return operators_; } static int check_mview_dep_session_vars(const ObTableSchema &mv_schema, const ObSQLSessionInfo &session, const bool gen_error, @@ -66,10 +77,11 @@ private: ObSQLSessionInfo &session_info, const uint64_t mv_id, const ObTableSchema *&mv_schema, + const ObTableSchema *&mv_container_schema, const ObSelectStmt *&view_stmt); TO_STRING_KV(K_(mview_id), K_(for_rt_expand), K_(inited), K_(refreshable_type), - K_(operators), K_(dependency_infos)); + K_(operators), K_(dependency_infos), K_(major_refresh_info)); private: common::ObArenaAllocator inner_alloc_; @@ -79,6 +91,7 @@ private: ObMVRefreshableType refreshable_type_; common::ObFixedArray operators_; // refresh or real time access operator for mv common::ObFixedArray dependency_infos_; + const MajorRefreshInfo *major_refresh_info_; }; } diff --git a/src/sql/resolver/ob_resolver_utils.cpp b/src/sql/resolver/ob_resolver_utils.cpp index 2268ae8f71..43e5793bf1 100644 --- a/src/sql/resolver/ob_resolver_utils.cpp +++ b/src/sql/resolver/ob_resolver_utils.cpp @@ -9722,7 +9722,7 @@ int ObResolverUtils::check_allowed_alter_operations_for_mlog( ret = OB_NOT_SUPPORTED; LOG_WARN("alter materialized view log is not supported", KR(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter materialized view log is"); - } else if (table_schema.has_mlog_table()) { + } else if (table_schema.required_by_mview_refresh()) { bool is_alter_pk = false; ObIndexArg::IndexActionType pk_action_type; for (int64_t i = 0; OB_SUCC(ret) && (i < arg.index_arg_list_.count()); ++i) { @@ -9776,40 +9776,50 @@ int ObResolverUtils::check_allowed_alter_operations_for_mlog( // generate more specific error messages if (is_alter_pk) { if (ObIndexArg::ADD_PRIMARY_KEY == pk_action_type) { - LOG_WARN("add primary key to table with materialized view log is not supported", + LOG_WARN( + "add primary key to table required by materialized view refresh is not supported", KR(ret), K(table_schema.get_table_name())); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "add primary key to table with materialized view log is"); + LOG_USER_ERROR(OB_NOT_SUPPORTED, + "add primary key to table required by materialized view refresh is"); } else if (ObIndexArg::DROP_PRIMARY_KEY == pk_action_type) { - LOG_WARN("drop the primary key of table with materialized view log is not supported", - KR(ret), K(table_schema.get_table_name())); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "drop the primary key of table with materialized view log is"); + LOG_WARN("drop the primary key of table required by materialized view refresh is not " + "supported", + KR(ret), K(table_schema.get_table_name())); + LOG_USER_ERROR(OB_NOT_SUPPORTED, + "drop the primary key of table required by materialized view refresh is"); } else { - LOG_WARN("alter the primary key of table with materialized view log is not supported", - KR(ret), K(table_schema.get_table_name())); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter the primary key of table with materialized view log is"); + LOG_WARN("alter the primary key of table required by materialized view refresh is not " + "supported", + KR(ret), K(table_schema.get_table_name())); + LOG_USER_ERROR(OB_NOT_SUPPORTED, + "alter the primary key of table required by materialized view refresh is"); } } else if (arg.is_alter_columns_) { - LOG_WARN("alter column of table with materialized view log is not supported", - KR(ret), K(table_schema.get_table_name())); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter column of table with materialized view log is"); + LOG_WARN("alter column of table required by materialized view refresh is not supported", + KR(ret), K(table_schema.get_table_name())); + LOG_USER_ERROR(OB_NOT_SUPPORTED, + "alter column of table required by materialized view refresh is"); } else if (arg.is_alter_partitions_) { - LOG_WARN("alter partition of table with materialized view log is not supported", - KR(ret), K(table_schema.get_table_name())); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter partition of table with materialized view log is"); + LOG_WARN("alter partition of table required by materialized view refresh is not supported", + KR(ret), K(table_schema.get_table_name())); + LOG_USER_ERROR(OB_NOT_SUPPORTED, + "alter partition of table required by materialized view refresh is"); } else if (arg.is_alter_options_) { if (arg.alter_table_schema_.alter_option_bitset_.has_member(ObAlterTableArg::TABLE_NAME)) { - LOG_WARN("alter name of table with materialized view log is not supported", - KR(ret), K(table_schema.get_table_name())); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter name of table with materialized view log is"); + LOG_WARN("alter name of table required by materialized view refresh is not supported", + KR(ret), K(table_schema.get_table_name())); + LOG_USER_ERROR(OB_NOT_SUPPORTED, + "alter name of table required by materialized view refresh is"); } else { - LOG_WARN("alter option of table with materialized view log is not supported", - KR(ret), K(table_schema.get_table_name()), K(arg)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter option of table with materialized view log is"); + LOG_WARN("alter option of table required by materialized view refresh is not supported", + KR(ret), K(table_schema.get_table_name()), K(arg)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, + "alter option of table required by materialized view refresh is"); } } else { - LOG_WARN("alter table with materialized view log is not supported", - KR(ret), K(table_schema.get_table_name()), K(arg)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter table with materialized view log is"); + LOG_WARN("alter table required by materialized view refresh is not supported", KR(ret), + K(table_schema.get_table_name()), K(arg)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter table required by materialized view refresh is"); } } } diff --git a/src/sql/rewrite/ob_transform_pre_process.cpp b/src/sql/rewrite/ob_transform_pre_process.cpp index 026cde453e..e95eff054c 100644 --- a/src/sql/rewrite/ob_transform_pre_process.cpp +++ b/src/sql/rewrite/ob_transform_pre_process.cpp @@ -374,6 +374,7 @@ int ObTransformPreProcess::expand_materialized_view(ObDMLStmt *stmt, bool &trans ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpect null", K(ret), K(stmt), K(ctx_)); } else if (ctx_->session_info_->get_ddl_info().is_refreshing_mview() + || ctx_->session_info_->get_ddl_info().is_major_refreshing_mview() || stmt->get_query_ctx()->get_global_hint().has_dbms_stats_hint()) { // 1. when refresh mview, do not expand rt-mv // 2. when gather stat, do not expand rt-mv diff --git a/src/sql/rewrite/ob_transform_utils.cpp b/src/sql/rewrite/ob_transform_utils.cpp index c4b650a124..043bc61ca2 100644 --- a/src/sql/rewrite/ob_transform_utils.cpp +++ b/src/sql/rewrite/ob_transform_utils.cpp @@ -1515,8 +1515,10 @@ int ObTransformUtils::update_table_id_for_pseudo_columns(const ObIArrayget_expr_type() - && T_ORA_ROWSCN == pseudo_columns.at(i)->get_expr_type()) { + } else if (other_pseudo_columns.at(i)->get_expr_type() != pseudo_columns.at(i)->get_expr_type()) { + /* do nothing */ + } else if (T_ORA_ROWSCN == pseudo_columns.at(i)->get_expr_type() + || T_PSEUDO_OLD_NEW_COL == pseudo_columns.at(i)->get_expr_type()) { ObPseudoColumnRawExpr *pseudo_col1 = static_cast(other_pseudo_columns.at(i)); ObPseudoColumnRawExpr *pseudo_col2 = static_cast(pseudo_columns.at(i)); if (pseudo_col1->get_table_id() == old_table_id) { @@ -7172,7 +7174,8 @@ int ObTransformUtils::check_need_pushdown_pseudo_column(const ObRawExpr &expr, int ret = OB_SUCCESS; need_pushdown = false; switch (expr.get_expr_type()) { - case T_ORA_ROWSCN: { + case T_ORA_ROWSCN: + case T_PSEUDO_OLD_NEW_COL: { need_pushdown = true; break; } diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 078f40dd63..7173e3c528 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -588,6 +588,7 @@ ob_set_subtarget(ob_storage access access/ob_multiple_scan_merge.cpp access/ob_multiple_skip_scan_merge.cpp access/ob_multiple_multi_skip_scan_merge.cpp + access/ob_multiple_mview_merge.cpp access/ob_table_scan_iterator.cpp access/ob_store_row_iterator.cpp access/ob_i_sample_iterator.cpp @@ -746,6 +747,7 @@ ob_set_subtarget(ob_storage compaction compaction/ob_sstable_merge_history.cpp compaction/ob_batch_freeze_tablets_dag.cpp compaction/ob_progressive_merge_helper.cpp + compaction/ob_mview_compaction_util.cpp ) ob_set_subtarget(ob_storage memtable diff --git a/src/storage/access/ob_dml_param.h b/src/storage/access/ob_dml_param.h index d6f149db90..4a4a787225 100644 --- a/src/storage/access/ob_dml_param.h +++ b/src/storage/access/ob_dml_param.h @@ -159,6 +159,9 @@ public: OB_INLINE bool use_index_skip_scan() const { return (1 == ss_key_ranges_.count()) && (!ss_key_ranges_.at(0).is_whole_range()); } + OB_INLINE bool is_mview_query() const { + return nullptr != op_filters_ && scan_flag_.is_mr_mview_query(); + } void destroy() override { if (OB_UNLIKELY(ss_key_ranges_.get_capacity() > OB_DEFAULT_RANGE_COUNT)) { diff --git a/src/storage/access/ob_fuse_row_cache_fetcher.cpp b/src/storage/access/ob_fuse_row_cache_fetcher.cpp index 7c6d07e7d0..ae0661cead 100644 --- a/src/storage/access/ob_fuse_row_cache_fetcher.cpp +++ b/src/storage/access/ob_fuse_row_cache_fetcher.cpp @@ -21,22 +21,34 @@ using namespace oceanbase::storage; using namespace oceanbase::blocksstable; ObFuseRowCacheFetcher::ObFuseRowCacheFetcher() - : is_inited_(false), tablet_id_(), read_info_(nullptr), tablet_version_(0) + : is_inited_(false), + type_(StorageScanType::NORMAL), + tablet_id_(), + read_info_(nullptr), + tablet_version_(0), + read_start_version_(0), + read_snapshot_version_(0) { } -int ObFuseRowCacheFetcher::init(const ObTabletID &tablet_id, +int ObFuseRowCacheFetcher::init(const StorageScanType type, + const ObTabletID &tablet_id, const ObITableReadInfo *read_info, - const int64_t tablet_version) + const int64_t tablet_version, + const int64_t read_start_version, + const int64_t read_snapshot_version) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!tablet_id.is_valid() || nullptr == read_info || tablet_version <= 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(tablet_id), KP(read_info), K(tablet_version)); } else { + type_ = type; tablet_id_ = tablet_id; read_info_ = read_info; tablet_version_ = tablet_version; + read_start_version_ = read_start_version; + read_snapshot_version_ = read_snapshot_version; is_inited_ = true; } return ret; @@ -52,6 +64,18 @@ int ObFuseRowCacheFetcher::get_fuse_row_cache(const ObDatumRowkey &rowkey, ObFus } else if (rowkey.get_datum_cnt() > read_info_->get_datum_utils().get_rowkey_count()) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "Unexpected invalid read info", K(ret), K(rowkey), KPC(read_info_)); + } else if (is_mview_table_scan(type_)) { + ObMultiVersionFuseRowCacheKey cache_key(read_start_version_, read_snapshot_version_, MTL_ID(), tablet_id_, rowkey, + read_info_->get_schema_column_count(), read_info_->get_datum_utils()); + if (OB_FAIL(ObStorageCacheSuite::get_instance().get_multi_version_fuse_row_cache().get_row(cache_key, handle))) { + if (OB_ENTRY_NOT_EXIST != ret) { + STORAGE_LOG(WARN, "fail to get row from multi version fuse row cache", K(ret), K(cache_key)); + } + } else { + // TODO(yht146439) add to sql_audit + EVENT_INC(ObStatEventIds::MULTI_VERSION_FUSE_ROW_CACHE_HIT); + } + STORAGE_LOG(DEBUG, "get from multi version fuse row cache", K(ret), K(cache_key)); } else { ObFuseRowCacheKey cache_key(MTL_ID(), tablet_id_, rowkey, tablet_version_, read_info_->get_schema_column_count(), read_info_->get_datum_utils()); if (OB_FAIL(ObStorageCacheSuite::get_instance().get_fuse_row_cache().get_row(cache_key, handle))) { @@ -66,28 +90,40 @@ int ObFuseRowCacheFetcher::get_fuse_row_cache(const ObDatumRowkey &rowkey, ObFus return ret; } -int ObFuseRowCacheFetcher::put_fuse_row_cache(const ObDatumRowkey &rowkey, ObDatumRow &row, const int64_t read_snapshot_version) +int ObFuseRowCacheFetcher::put_fuse_row_cache(const ObDatumRowkey &rowkey, ObDatumRow &row) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObFuseRowCacheFetcher has not been inited", K(ret)); - } else if (OB_UNLIKELY(read_snapshot_version <= 0)) { + } else if (OB_UNLIKELY(read_snapshot_version_ <= 0)) { ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument to put fuse row cache", K(ret), K(read_snapshot_version)); + STORAGE_LOG(WARN, "Invalid argument to put fuse row cache", K(ret), K(read_snapshot_version_)); } else if (row.snapshot_version_ == INT64_MAX) { // uncommited row value, do not put into row cache + } else if (is_mview_table_scan(type_)) { + int tmp_ret = OB_SUCCESS; + ObMultiVersionFuseRowCacheKey cache_key(read_start_version_, read_snapshot_version_, MTL_ID(), tablet_id_, rowkey, + read_info_->get_schema_column_count(), read_info_->get_datum_utils()); + ObFuseRowCacheValue row_cache_value; + if (OB_SUCCESS != (tmp_ret = row_cache_value.init(row, read_snapshot_version_))) { + STORAGE_LOG(WARN, "fail to init row cache value", K(tmp_ret)); + } else if (OB_SUCCESS != (tmp_ret = ObStorageCacheSuite::get_instance().get_multi_version_fuse_row_cache().put_row(cache_key, row_cache_value))) { + STORAGE_LOG(WARN, "fail to put row into multi version fuse row cache", K(tmp_ret)); + } else { + STORAGE_LOG(DEBUG, "update multi version fuse row cache", K(cache_key), K(row_cache_value), K(row), KPC(read_info_)); + } } else { // update row cache int tmp_ret = OB_SUCCESS; ObFuseRowCacheKey cache_key(MTL_ID(), tablet_id_, rowkey, tablet_version_, read_info_->get_schema_column_count(), read_info_->get_datum_utils()); ObFuseRowCacheValue row_cache_value; - if (OB_SUCCESS != (tmp_ret = row_cache_value.init(row, read_snapshot_version))) { + if (OB_SUCCESS != (tmp_ret = row_cache_value.init(row, read_snapshot_version_))) { STORAGE_LOG(WARN, "fail to init row cache value", K(tmp_ret)); } else if (OB_SUCCESS != (tmp_ret = ObStorageCacheSuite::get_instance().get_fuse_row_cache().put_row(cache_key, row_cache_value))) { STORAGE_LOG(WARN, "fail to put row into fuse row cache", K(tmp_ret)); } else { - STORAGE_LOG(DEBUG, "update row cache", K(cache_key), K(row_cache_value), K(row), KPC(read_info_)); + STORAGE_LOG(DEBUG, "update fuse row cache", K(cache_key), K(row_cache_value), K(row), KPC(read_info_)); } } diff --git a/src/storage/access/ob_fuse_row_cache_fetcher.h b/src/storage/access/ob_fuse_row_cache_fetcher.h index 415c24b334..f5e1365043 100644 --- a/src/storage/access/ob_fuse_row_cache_fetcher.h +++ b/src/storage/access/ob_fuse_row_cache_fetcher.h @@ -29,14 +29,18 @@ class ObFuseRowCacheFetcher final public: ObFuseRowCacheFetcher(); ~ObFuseRowCacheFetcher() = default; - int init(const ObTabletID &tablet_id, const ObITableReadInfo *read_info, const int64_t tablet_version); + int init(const StorageScanType type, const ObTabletID &tablet_id, const ObITableReadInfo *read_info, const int64_t tablet_version, + const int64_t read_start_version, const int64_t read_snapshot_version); int get_fuse_row_cache(const blocksstable::ObDatumRowkey &rowkey, blocksstable::ObFuseRowValueHandle &handle); - int put_fuse_row_cache(const blocksstable::ObDatumRowkey &rowkey, blocksstable::ObDatumRow &row, const int64_t read_snapshot_version); + int put_fuse_row_cache(const blocksstable::ObDatumRowkey &rowkey, blocksstable::ObDatumRow &row); private: bool is_inited_; + StorageScanType type_; ObTabletID tablet_id_; const ObITableReadInfo *read_info_; int64_t tablet_version_; + int64_t read_start_version_; + int64_t read_snapshot_version_; }; } // end namespace storage diff --git a/src/storage/access/ob_multiple_get_merge.cpp b/src/storage/access/ob_multiple_get_merge.cpp index 6fb0e6a073..1b8094c316 100644 --- a/src/storage/access/ob_multiple_get_merge.cpp +++ b/src/storage/access/ob_multiple_get_merge.cpp @@ -218,7 +218,7 @@ int ObMultipleGetMerge::inner_get_next_row(ObDatumRow &row) } if (OB_SUCCESS == ret) { ++get_row_range_idx_; - if (fuse_row.row_flag_.is_exist_without_delete()) { + if (fuse_row.row_flag_.is_exist_without_delete() || (iter_del_row_ && fuse_row.row_flag_.is_delete())) { if (get_row_range_idx_ > rowkeys_->count()) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "unexptected range idx", K(ret), K(get_row_range_idx_), K(rowkeys_->count())); diff --git a/src/storage/access/ob_multiple_merge.cpp b/src/storage/access/ob_multiple_merge.cpp index 9efa5f93d2..05996e683c 100644 --- a/src/storage/access/ob_multiple_merge.cpp +++ b/src/storage/access/ob_multiple_merge.cpp @@ -61,6 +61,7 @@ ObMultipleMerge::ObMultipleMerge() need_fill_virtual_columns_(false), need_output_row_with_nop_(false), inited_(false), + iter_del_row_(false), range_idx_delta_(0), get_table_param_(nullptr), read_memtable_only_(false), @@ -951,6 +952,7 @@ void ObMultipleMerge::inner_reset() read_memtable_only_ = false; lob_reader_.reset(); scan_state_ = ScanState::NONE; + iter_del_row_ = false; } void ObMultipleMerge::reset_iter_array(const bool can_reuse) @@ -1342,10 +1344,8 @@ int ObMultipleMerge::prepare_read_tables(bool refresh) LOG_WARN("get table iterator fail", K(ret), K_(get_table_param), KP_(access_param)); } } else if (OB_FAIL(get_table_param_->tablet_iter_.refresh_read_tables_from_tablet( - get_table_param_->sample_info_.is_no_sample() - ? access_ctx_->store_ctx_->mvcc_acc_ctx_.get_snapshot_version().get_val_for_tx() - : INT64_MAX, - false/*allow_not_ready*/, + generate_read_tables_version(), + false/*allow_not_ready*/, false/*major_sstable_only*/, need_split_src_table, need_split_dst_table))) { @@ -1393,6 +1393,7 @@ int ObMultipleMerge::prepare_tables_from_iterator(ObTableStoreIterator &table_it int64_t memtable_cnt = 0; read_memtable_only_ = false; bool read_released_memtable = false; + int64_t major_version = -1; while (OB_SUCC(ret)) { ObITable *table_ptr = nullptr; bool need_table = true; @@ -1413,6 +1414,9 @@ int ObMultipleMerge::prepare_tables_from_iterator(ObTableStoreIterator &table_it need_table = !table_ptr->is_major_sstable(); } } + if (OB_SUCC(ret)) { + need_table = check_table_need_read(*table_ptr, major_version); + } if (OB_SUCC(ret) && need_table) { if (table_ptr->no_data_to_read()) { LOG_DEBUG("cur table is empty", K(ret), KPC(table_ptr)); @@ -1506,9 +1510,7 @@ int ObMultipleMerge::refresh_tablet_iter() const int64_t remain_timeout = THIS_WORKER.get_timeout_remain(); const share::ObLSID &ls_id = access_ctx_->ls_id_; const common::ObTabletID &tablet_id = get_table_param_->tablet_iter_.get_tablet()->get_tablet_meta().tablet_id_; - const int64_t snapshot_version = get_table_param_->sample_info_.is_no_sample() - ? access_ctx_->store_ctx_->mvcc_acc_ctx_.get_snapshot_version().get_val_for_tx() - : INT64_MAX; + const int64_t snapshot_version = generate_read_tables_version(); if (OB_UNLIKELY(remain_timeout <= 0)) { ret = OB_TIMEOUT; LOG_WARN("timeout reached", K(ret), K(ls_id), K(tablet_id), K(remain_timeout)); @@ -1734,7 +1736,8 @@ int ObMultipleMerge::set_base_version() const { int ret = OB_SUCCESS; // When the major table is currently being processed, the snapshot version is taken and placed // in the current context for base version to filter unnecessary rows in the mini or minor sstable - if (!access_param_->iter_param_.is_skip_scan() && is_scan() && tables_.count() > 1) { + if (!access_param_->iter_param_.is_skip_scan() && + !access_ctx_->is_mview_query() && is_scan() && tables_.count() > 1) { ObITable *table = nullptr; if (OB_FAIL(tables_.at(0, table))) { STORAGE_LOG(WARN, "Fail to get the first store", K(ret)); @@ -1745,5 +1748,17 @@ int ObMultipleMerge::set_base_version() const { return ret; } +int64_t ObMultipleMerge::generate_read_tables_version() const +{ + return get_table_param_->sample_info_.is_no_sample() ? + access_ctx_->store_ctx_->mvcc_acc_ctx_.get_snapshot_version().get_val_for_tx() : INT64_MAX; +} + +bool ObMultipleMerge::check_table_need_read(const ObITable &table, int64_t &major_version) const +{ + UNUSEDx(table, major_version); + return true; +} + } } diff --git a/src/storage/access/ob_multiple_merge.h b/src/storage/access/ob_multiple_merge.h index b45f6fea7d..e80300ebf4 100644 --- a/src/storage/access/ob_multiple_merge.h +++ b/src/storage/access/ob_multiple_merge.h @@ -59,6 +59,8 @@ public: virtual void reuse(); // used for global cached query iterator virtual void reclaim(); + // used for mview table scan + virtual int open(ObTableScanRange &table_scan_range) { return OB_NOT_SUPPORTED; } void disable_padding() { need_padding_ = false; } void disable_fill_default() { need_fill_default_ = false; } @@ -67,7 +69,9 @@ public: OB_INLINE bool is_inited() { return inited_; } OB_INLINE bool is_read_memtable_only() const { return read_memtable_only_; } OB_INLINE const common::ObIArray &get_out_project_cells() { return out_project_cols_; } - + OB_INLINE ObNopPos &get_nop_pos() { return nop_pos_; } + OB_INLINE void set_iter_del_row(const bool iter_del_row) { iter_del_row_ = iter_del_row; } + OB_INLINE blocksstable::ObDatumRow &get_unprojected_row() { return unprojected_row_; } protected: int open(); virtual int calc_scan_range() = 0; @@ -77,6 +81,8 @@ protected: virtual int inner_get_next_row(blocksstable::ObDatumRow &row) = 0; virtual int inner_get_next_rows() { return OB_SUCCESS; }; virtual int can_batch_scan(bool &can_batch) { can_batch = false; return OB_SUCCESS; } + virtual int64_t generate_read_tables_version() const; + virtual bool check_table_need_read(const ObITable &table, int64_t &major_version) const; int add_iterator(ObStoreRowIterator &iter); // for unit test const ObTableIterParam * get_actual_iter_param(const ObITable *table) const; int project_row(const blocksstable::ObDatumRow &unprojected_row, @@ -144,6 +150,7 @@ protected: bool need_fill_virtual_columns_; // disabled by join mv scan bool need_output_row_with_nop_; // for sampling increment data bool inited_; + bool iter_del_row_; int64_t range_idx_delta_; ObGetTableParam *get_table_param_; bool read_memtable_only_; diff --git a/src/storage/access/ob_multiple_mview_merge.cpp b/src/storage/access/ob_multiple_mview_merge.cpp new file mode 100644 index 0000000000..a64ce676a7 --- /dev/null +++ b/src/storage/access/ob_multiple_mview_merge.cpp @@ -0,0 +1,456 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE +#include "ob_multiple_mview_merge.h" + +namespace oceanbase +{ +namespace storage +{ + +int ObBaseTableAccessInfo::construct_access_ctx( + common::ObIAllocator *range_allocator, + const ObTableAccessContext &ctx) +{ + int ret = OB_SUCCESS; + common::ObVersionRange trans_version_range; + trans_version_range.snapshot_version_ = ctx.trans_version_range_.base_version_; + trans_version_range.multi_version_start_ = 0; + trans_version_range.base_version_ = 0; + share::SCN tmp_scn; + if (OB_FAIL(tmp_scn.convert_for_tx(trans_version_range.snapshot_version_))) { + LOG_WARN("Failed to convert scn", K(ret), K(trans_version_range)); + } else if (FALSE_IT(store_ctx_.reset())) { + } else if (OB_FAIL(store_ctx_.init_for_read(ctx.ls_id_, ctx.tablet_id_, INT64_MAX, -1, tmp_scn))) { + LOG_WARN("Failed to init store ctx", K(ret), K(ctx)); + } else if (FALSE_IT(store_ctx_.mvcc_acc_ctx_.tx_desc_ = ctx.store_ctx_->mvcc_acc_ctx_.tx_desc_)) { + } else if (FALSE_IT(access_ctx_.reuse())) { + } else if (OB_FAIL(access_ctx_.init_for_mview(range_allocator, ctx, store_ctx_))) { + LOG_WARN("Failed to init access ctx", K(ret), K(ctx)); + } + return ret; +} + +ObMviewBaseMerge::ObMviewBaseMerge() + : ObSingleMerge() +{ +} + +ObMviewBaseMerge::~ObMviewBaseMerge() +{ +} + +bool ObMviewBaseMerge::check_table_need_read(const ObITable &table, int64_t &major_version) const +{ + bool need_read = true; + if (table.is_major_sstable()) { + major_version = table.get_snapshot_version(); + } else if (major_version > 0) { + need_read = major_version != access_ctx_->trans_version_range_.snapshot_version_; + } + return need_read; +} + +ObMviewIncrMerge::ObMviewIncrMerge() + : base_data_merge_(nullptr), + base_access_info_(nullptr), + is_table_store_refreshed_(false), + scan_old_row_(false), + col_descs_(nullptr), + base_rowkey_(), + rowkey_allocator_("ObMview", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()) +{ +} + +ObMviewIncrMerge::~ObMviewIncrMerge() +{ + reset(); +} + +void ObMviewIncrMerge::reset() +{ + is_table_store_refreshed_ = false; + scan_old_row_ = false; + common::ObIAllocator *alloc = nullptr == base_access_info_ ? + nullptr : base_access_info_->access_ctx_.stmt_allocator_; + if (OB_NOT_NULL(base_data_merge_)) { + base_data_merge_->~ObMviewBaseMerge(); + if (OB_NOT_NULL(alloc)) { + alloc->free(base_data_merge_); + } + base_data_merge_ = nullptr; + } + if (OB_NOT_NULL(base_access_info_)) { + base_access_info_->~ObBaseTableAccessInfo(); + if (OB_NOT_NULL(alloc)) { + alloc->free(base_access_info_); + } + base_access_info_ = nullptr; + } + col_descs_ = nullptr; + base_rowkey_.reset(); + rowkey_allocator_.reset(); +} + +void ObMviewIncrMerge::reuse() +{ + if (nullptr != base_data_merge_) { + base_data_merge_->reuse(); + } + rowkey_allocator_.reuse(); +} + +int ObMviewIncrMerge::get_incr_row( + ObTableAccessParam ¶m, + ObTableAccessContext &context, + ObGetTableParam &get_table_param, + ObMultipleMerge &merge, + blocksstable::ObDatumRow &row) +{ + int ret = OB_SUCCESS; + bool version_fit = false; + bool read_row = false; + const bool version_include_base = 0 == context.trans_version_range_.base_version_; + const StorageScanType scan_type = context.mview_scan_info_->scan_type_; + while (OB_SUCC(ret) && !read_row) { + version_fit = false; + row.count_ = 0; + row.row_flag_.set_flag(ObDmlFlag::DF_NOT_EXIST); + if (OB_FAIL(get_storage_row(row))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("Failed to get merged row", K(ret)); + } + } else if (OB_FAIL(check_version_fit(param, context, row, version_fit))) { + LOG_WARN("Failed to check version fit", K(ret)); + } else if (!version_fit) { + } else if (FALSE_IT(scan_old_row_ = is_mview_scan_old_row(scan_type) || (is_mview_scan_final_row(scan_type) && row.row_flag_.is_delete()))) { + } else if (version_include_base && scan_old_row_) { + ret = OB_ITER_END; + } else if (version_include_base || (!scan_old_row_ && has_no_nop_values(param, merge.get_nop_pos()))) { + if (row.row_flag_.is_exist_without_delete()) { + LOG_DEBUG("output insert row", K(ret), K(row), K(version_include_base), K_(scan_old_row), + K(merge.get_nop_pos().count())); + read_row = true; + } + } else { + if (OB_FAIL(base_rowkey_.assign(row.storage_datums_, param.iter_param_.get_schema_rowkey_count()))) { + LOG_WARN("Failed to assign incr rowkey", K(ret)); + } else if (OB_FAIL(open_base_data_merge(param, context, get_table_param, base_rowkey_))) { + LOG_WARN("Failed to open base data merge", K(ret)); + } else { + blocksstable::ObDatumRow &base_row = base_data_merge_->get_unprojected_row(); + base_row.count_ = 0; + base_row.row_flag_.set_flag(ObDmlFlag::DF_NOT_EXIST); + if (base_data_merge_->is_empty()) { + } else if (OB_FAIL(base_data_merge_->get_storage_row(base_row))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("Failed to get base data row", K(ret)); + } else if (OB_UNLIKELY(row.row_flag_.is_exist_without_delete() && !has_no_nop_values(param, merge.get_nop_pos()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Base row is not found", K(ret), K(row), K(merge.get_nop_pos())); + } else { + ret = OB_SUCCESS; + } + } else { + LOG_DEBUG("[MVIEW QUERY]: get base row", K(ret), K(base_row), K(version_include_base), K_(scan_old_row)); + } + if (FAILEDx(generate_output_row(param, merge.get_nop_pos(), base_row, row))) { + LOG_WARN("Failed to generate row", K(ret)); + } else { + read_row = row.row_flag_.is_exist_without_delete(); + } + } + } + } + if (FAILEDx(set_old_new_row_flag(param, context, row))) { + LOG_WARN("Failed to set old new row flag", K(ret)); + } + LOG_DEBUG("[MVIEW QUERY]: get next row", K(ret), K(row), K(version_include_base), K_(scan_old_row)); + return ret; +} + +int ObMviewIncrMerge::check_version_fit( + const ObTableAccessParam ¶m, + const ObTableAccessContext &context, + const blocksstable::ObDatumRow &row, + bool &version_fit) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!param.iter_param_.need_scn_ || !row.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid scn or row", K(ret), K(param.iter_param_), K(row)); + } else { + const int64_t trans_idx = param.iter_param_.get_read_info()->get_trans_col_index(); + version_fit = row.storage_datums_[trans_idx].get_int() > context.trans_version_range_.base_version_; + LOG_DEBUG("[MVIEW QUERY]: check incr row version", K(ret), K(trans_idx), K(version_fit), + K(row.storage_datums_[trans_idx].get_int()), K(context.trans_version_range_.base_version_)); + } + return ret; +} + +int ObMviewIncrMerge::generate_output_row( + const ObTableAccessParam ¶m, + ObNopPos &nop_pos, + const blocksstable::ObDatumRow &base_row, + blocksstable::ObDatumRow &row) +{ + int ret = OB_SUCCESS; + bool should_output = false; + if (OB_UNLIKELY(!row.is_valid() || + !row.row_flag_.is_exist())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid incr row flag", K(ret), K(base_row), K(row)); + } else if (FALSE_IT(row.row_flag_.set_flag(ObDmlFlag::DF_INSERT))) { + } else { + // TableScan1(MVIEW_FIRST_DELETE), output DELETE old row + // TableScan2(MVIEW_LAST_INSERT), output INSERT new row + // if base_row.row_flag_.is_exist_without_delete() is true, means that the old base row is exist and + // TableScan is MVIEW_FIRST_DELETE, incr row is (DELETE/INSERT/UPDATE), should output old base row + // TableScan is MVIEW_LAST_INSERT, then DELETE row is ignored internal, incr row is(INSERT/UPDATE), should output new incr row + // if base_row.row_flag_.is_exist_without_delete() is false, means that the old base row is not exist and + // TableScan is MVIEW_FIRST_DELETE, no need to ouput + // TableScan is MVIEW_LAST_INSERT, then DELETE row is ignored internal, incr row is(INSERT/UPDATE), should output new incr row + should_output = base_row.row_flag_.is_exist_without_delete() || !scan_old_row_; + } + if (OB_FAIL(ret)) { + } else if (should_output) { + bool final_result = false; + if (scan_old_row_) { + const int64_t trans_idx = param.iter_param_.get_read_info()->get_trans_col_index(); + for (int64_t i = 0; i < base_row.count_; ++i) { + if (OB_LIKELY(i != trans_idx)) { + row.storage_datums_[i] = base_row.storage_datums_[i]; + } + } + final_result = true; + } else if (has_no_nop_values(param, nop_pos)) { + } else if (OB_FAIL(ObRowFuse::fuse_row(base_row, row, nop_pos, final_result))) { + LOG_WARN("Failed to fuse incr row", K(ret), K(base_row), K(row)); + } + } else { + row.row_flag_.set_flag(ObDmlFlag::DF_NOT_EXIST); + LOG_DEBUG("[MVIEW QUERY]: no need to output row", K(ret), K(base_row), K(row), K_(scan_old_row)); + } + LOG_DEBUG("[MVIEW QUERY]: generate row", K(ret), K(base_row), K(row), K_(scan_old_row)); + return ret; +} + +int ObMviewIncrMerge::open_base_data_merge( + ObTableAccessParam ¶m, + ObTableAccessContext &context, + ObGetTableParam &get_table_param, + const ObDatumRowkey &rowkey) +{ + int ret = OB_SUCCESS; + if (nullptr == base_data_merge_) { + is_table_store_refreshed_ = false; + if (OB_ISNULL(base_access_info_ = OB_NEWx(ObBaseTableAccessInfo, context.stmt_allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("Failed to alloc base access info", K(ret)); + } else if (OB_ISNULL(base_data_merge_ = OB_NEWx(ObMviewBaseMerge, context.stmt_allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("Failed to alloc base data merge", K(ret)); + } else if (OB_FAIL(base_access_info_->construct_access_ctx(&rowkey_allocator_, context))) { + LOG_WARN("Failed to cons version range and ctx", K(ret)); + } else if (OB_FAIL(base_data_merge_->init(param, base_access_info_->access_ctx_, get_table_param))) { + LOG_WARN("Failed to init base data merge", K(ret)); + } else if (OB_ISNULL(col_descs_ = param.iter_param_.get_out_col_descs())) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "Unexpected null out cols", K(ret)); + } + } else { + base_data_merge_->reuse(); + rowkey_allocator_.reuse(); + } + if (OB_FAIL(ret)) { + } else if (is_table_store_refreshed_) { + LOG_INFO("table store refreshed", K(is_table_store_refreshed_), K(param.iter_param_.tablet_id_)); + if (OB_FAIL(base_access_info_->construct_access_ctx(&rowkey_allocator_, context))) { + LOG_WARN("Failed to cons version range and ctx", K(ret)); + } else if (OB_FAIL(base_data_merge_->switch_param(param, base_access_info_->access_ctx_, get_table_param))) { + LOG_WARN("Failed to switch param", K(ret)); + } else { + is_table_store_refreshed_ = false; + } + } + if (OB_FAIL(ret)) { + } else if (base_data_merge_->is_empty()) { + LOG_TRACE("[MVIEW QUERY]: base data is empty", K(ret)); + } else if (OB_FAIL(base_rowkey_.prepare_memtable_readable(*col_descs_, rowkey_allocator_))) { + LOG_WARN("Failed to prepare memtable rowkey", K(ret)); + } else if (OB_FAIL(base_data_merge_->open(rowkey))) { + LOG_WARN("Failed to open base data merge", K(ret)); + } + return ret; +} + +int ObMviewIncrMerge::set_old_new_row_flag( + const ObTableAccessParam ¶m, + const ObTableAccessContext &context, + blocksstable::ObDatumRow &row) const +{ + int ret = OB_SUCCESS; + const int64_t old_new_idx = param.iter_param_.get_mview_old_new_col_index(); + if (OB_UNLIKELY(OB_INVALID_INDEX == old_new_idx)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid mview old new col index", K(ret), K(old_new_idx)); + } else if (scan_old_row_) { + row.storage_datums_[old_new_idx].set_string(ObMviewScanInfo::OLD_ROW); + } else { + row.storage_datums_[old_new_idx].set_string(ObMviewScanInfo::NEW_ROW); + } + return ret; +} + +int ObMviewSingleMerge::open(ObTableScanRange &table_scan_range) +{ + return ObSingleMerge::open(table_scan_range.get_rowkeys().at(0)); +} + +int ObMviewScanMerge::open(ObTableScanRange &table_scan_range) +{ + return ObMultipleScanMerge::open(table_scan_range.get_ranges().at(0)); +} + +int ObMviewGetMerge::open(ObTableScanRange &table_scan_range) +{ + return ObMultipleGetMerge::open(table_scan_range.get_rowkeys()); +} + +int ObMviewMultiScanMerge::open(ObTableScanRange &table_scan_range) +{ + return ObMultipleMultiScanMerge::open(table_scan_range.get_ranges()); +} + +ObMviewMergeWrapper::ObMviewMergeWrapper() +{ + MEMSET(merges_, 0, sizeof(merges_)); +} + +ObMviewMergeWrapper::~ObMviewMergeWrapper() +{ + for (int64_t i = 0; i < T_MAX_ITER_TYPE; ++i) { + if (nullptr != merges_[i]) { + merges_[i]->~ObMviewMerge(); + merges_[i] = nullptr; + } + } +} + +void ObMviewMergeWrapper::reuse() +{ + for (int64_t i = 0; i < T_MAX_ITER_TYPE; ++i) { + if (nullptr != merges_[i]) { + merges_[i]->reuse(); + } + } +} + +int ObMviewMergeWrapper::switch_param( + ObTableAccessParam ¶m, + ObTableAccessContext &context, + ObGetTableParam &get_table_param) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < T_MAX_ITER_TYPE; ++i) { + if (nullptr != merges_[i] && + OB_FAIL(merges_[i]->switch_param(param, context, get_table_param))) { + LOG_WARN("Failed to switch param", K(ret), K(i)); + } + } + return ret; +} + +int ObMviewMergeWrapper::alloc_mview_merge( + ObTableAccessParam ¶m, + ObTableAccessContext &context, + ObGetTableParam &get_table_param, + ObTableScanRange &table_scan_range, + ObMviewMergeWrapper *&merge_wrapper, + ObMviewMerge *&mview_merge) +{ + int ret = OB_SUCCESS; + common::ObIAllocator *alloctor = context.stmt_allocator_; + if (nullptr == merge_wrapper && + OB_ISNULL(merge_wrapper = OB_NEWx(ObMviewMergeWrapper, alloctor))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("Failed to alloc mview merge wrapper", K(ret)); + } else if (OB_FAIL(merge_wrapper->get_mview_merge(param, context, get_table_param, table_scan_range, mview_merge))) { + LOG_WARN("Failed to get mview merge", K(ret)); + } + return ret; +} + +int ObMviewMergeWrapper::get_mview_merge( + ObTableAccessParam ¶m, + ObTableAccessContext &context, + ObGetTableParam &get_table_param, + ObTableScanRange &table_scan_range, + ObMviewMerge *&version_merge) +{ + int ret = OB_SUCCESS; + ObQRIterType merge_type; + ObMviewMerge *tmp_merge = nullptr; + version_merge = nullptr; + if (OB_FAIL(table_scan_range.get_query_iter_type(merge_type))) { + LOG_WARN("Failed to get query iter type", K(ret)); + } else if (nullptr == (tmp_merge = merges_[merge_type])) { + switch (merge_type) { + case T_SINGLE_GET: { + context.use_fuse_row_cache_ = context.mview_scan_info_->is_mv_refresh_query_; + tmp_merge = OB_NEWx(ObMviewSingleMerge, context.stmt_allocator_); + break; + } + case T_SINGLE_SCAN: { + tmp_merge = OB_NEWx(ObMviewScanMerge, context.stmt_allocator_); + break; + } + case T_MULTI_GET: { + tmp_merge = OB_NEWx(ObMviewGetMerge, context.stmt_allocator_); + break; + } + case T_MULTI_SCAN: { + tmp_merge = OB_NEWx(ObMviewMultiScanMerge, context.stmt_allocator_); + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected merge type", K(ret), K(merge_type)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(tmp_merge)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("Failed to alloc merge", K(ret)); + } else if (OB_FAIL(tmp_merge->init(param, context, get_table_param))) { + LOG_WARN("Failed to init merge", K(ret)); + } + LOG_DEBUG("[MVIEW QUERY]: get version range merge", K(ret), K(merge_type), KP(tmp_merge)); + } + if (FAILEDx(tmp_merge->open(table_scan_range))) { + LOG_WARN("Failed to open mview merge", K(ret)); + } else { + merges_[merge_type] = tmp_merge; + version_merge = tmp_merge; + version_merge->set_iter_del_row(is_mview_need_deleted_row(context.mview_scan_info_->scan_type_)); + param.op_filters_ = &context.mview_scan_info_->op_filters_; + } + if (OB_FAIL(ret) && nullptr != tmp_merge) { + tmp_merge->~ObMviewMerge(); + context.stmt_allocator_->free(tmp_merge); + } + return ret; +} + +} +} \ No newline at end of file diff --git a/src/storage/access/ob_multiple_mview_merge.h b/src/storage/access/ob_multiple_mview_merge.h new file mode 100644 index 0000000000..112dbc84e1 --- /dev/null +++ b/src/storage/access/ob_multiple_mview_merge.h @@ -0,0 +1,196 @@ +// Copyright (c) 2024 OceanBase +// OceanBase is licensed under Mulan PubL v2. +// You can use this software according to the terms and conditions of the Mulan PubL v2. +// You may obtain a copy of Mulan PubL v2 at: +// http://license.coscl.org.cn/MulanPubL-2.0 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +// EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +// MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PubL v2 for more details. + +#ifndef OCEANBASE_STORAGE_OB_MULTIPLE_MVIEW_MERGE_ +#define OCEANBASE_STORAGE_OB_MULTIPLE_MVIEW_MERGE_ +#include "ob_multiple_merge.h" +#include "ob_single_merge.h" +#include "ob_multiple_scan_merge.h" +#include "ob_multiple_get_merge.h" +#include "ob_multiple_multi_scan_merge.h" + +namespace oceanbase +{ +namespace storage +{ + +struct ObBaseTableAccessInfo +{ + int construct_access_ctx( + common::ObIAllocator *range_allocator, + const ObTableAccessContext &ctx); + ObStoreCtx store_ctx_; + ObTableAccessContext access_ctx_; +}; + +// base iterator to get base row before version range +// (0, base_version] +class ObMviewBaseMerge final : public ObSingleMerge +{ +public: + ObMviewBaseMerge(); + virtual ~ObMviewBaseMerge(); + OB_INLINE int get_storage_row(blocksstable::ObDatumRow &row) + { + return ObSingleMerge::inner_get_next_row(row); + } + OB_INLINE bool is_empty() const + { + return tables_.empty(); + } + virtual bool check_table_need_read(const ObITable &table, int64_t &major_version) const override; +}; + +// incremental iterator to get incremental row in version range +// (base_version, read snapshot version] +// use ObMviewBaseMerge to check old row's exists and fuse nop values +class ObMviewIncrMerge +{ +public: + ObMviewIncrMerge(); + virtual ~ObMviewIncrMerge(); + virtual int get_storage_row(blocksstable::ObDatumRow &row) = 0; + void reset(); + void reuse(); +protected: + int get_incr_row( + ObTableAccessParam ¶m, + ObTableAccessContext &context, + ObGetTableParam &get_table_param, + ObMultipleMerge &merge, + blocksstable::ObDatumRow &row); + ObMviewBaseMerge *base_data_merge_; + ObBaseTableAccessInfo *base_access_info_; + bool is_table_store_refreshed_; +private: + int check_version_fit( + const ObTableAccessParam ¶m, + const ObTableAccessContext &context, + const blocksstable::ObDatumRow &row, + bool &version_fit); + int generate_output_row( + const ObTableAccessParam ¶m, + ObNopPos &nop_pos, + const blocksstable::ObDatumRow &base_row, + blocksstable::ObDatumRow &row); + int open_base_data_merge( + ObTableAccessParam ¶m, + ObTableAccessContext &context, + ObGetTableParam &get_table_param, + const ObDatumRowkey &rowkey); + int set_old_new_row_flag( + const ObTableAccessParam ¶m, + const ObTableAccessContext &context, + blocksstable::ObDatumRow &row) const; + OB_INLINE bool has_no_nop_values(const ObTableAccessParam ¶m, const ObNopPos &nop_pos) const + { + const int64_t old_new_idx = param.iter_param_.get_mview_old_new_col_index(); + const int64_t group_idx = param.iter_param_.get_group_idx_col_index(); + return 0 == nop_pos.count() || + (1 == nop_pos.count() && nop_pos.nops_[0] == old_new_idx) || + (2 == nop_pos.count() && ((nop_pos.nops_[0] == old_new_idx && nop_pos.nops_[1] == group_idx) || + (nop_pos.nops_[0] == group_idx && nop_pos.nops_[1] == old_new_idx))); + } + bool scan_old_row_; + const ObColDescIArray *col_descs_; + ObDatumRowkey base_rowkey_; + common::ObArenaAllocator rowkey_allocator_; +}; + +#define DEFINE_MERGE_WRAPPER(Classname, BaseClass) \ +class Classname final : public BaseClass, public ObMviewIncrMerge { \ +public: \ + virtual int switch_param(ObTableAccessParam ¶m, ObTableAccessContext &context, \ + ObGetTableParam &get_table_param) override \ + { \ + int ret = OB_SUCCESS; \ + if (OB_FAIL(BaseClass::switch_param(param, context, get_table_param))) { \ + STORAGE_LOG(WARN, "Failed to switch param", K(ret)); \ + } else if (nullptr != base_data_merge_) { \ + is_table_store_refreshed_ = true; \ + } \ + return ret; \ + } \ + virtual int open(ObTableScanRange &table_scan_range) override; \ + virtual int get_storage_row(blocksstable::ObDatumRow &row) override \ + { \ + return BaseClass::inner_get_next_row(row); \ + } \ + virtual void reset() override \ + { \ + BaseClass::reset(); \ + ObMviewIncrMerge::reset(); \ + } \ + virtual void reuse() override \ + { \ + BaseClass::reuse(); \ + ObMviewIncrMerge::reuse(); \ + } \ +protected: \ + virtual int inner_get_next_row(blocksstable::ObDatumRow &row) override \ + { \ + return ObMviewIncrMerge::get_incr_row(*access_param_, *access_ctx_, *get_table_param_, *this, row);\ + } \ + virtual int calc_scan_range() override \ + { \ + is_table_store_refreshed_ = true; \ + return BaseClass::calc_scan_range(); \ + } \ +private: \ + virtual int64_t generate_read_tables_version() const override \ + { \ + return access_ctx_->trans_version_range_.base_version_ > 0 ? \ + access_ctx_->trans_version_range_.base_version_ : \ + access_ctx_->store_ctx_->mvcc_acc_ctx_.get_snapshot_version().get_val_for_tx(); \ + } \ + virtual bool check_table_need_read(const ObITable &table, int64_t &major_version) const override \ + { \ + return !table.is_major_sstable(); \ + } \ +}; + +DEFINE_MERGE_WRAPPER(ObMviewSingleMerge, ObSingleMerge); +DEFINE_MERGE_WRAPPER(ObMviewScanMerge, ObMultipleScanMerge); +DEFINE_MERGE_WRAPPER(ObMviewGetMerge, ObMultipleGetMerge); +DEFINE_MERGE_WRAPPER(ObMviewMultiScanMerge, ObMultipleMultiScanMerge); +#undef DEFINE_MERGE_WRAPPER + +typedef ObMultipleMerge ObMviewMerge; +class ObMviewMergeWrapper +{ +public: + ObMviewMergeWrapper(); + ~ObMviewMergeWrapper(); + void reuse(); + int switch_param( + ObTableAccessParam ¶m, + ObTableAccessContext &context, + ObGetTableParam &get_table_param); + static int alloc_mview_merge( + ObTableAccessParam ¶m, + ObTableAccessContext &context, + ObGetTableParam &get_table_param, + ObTableScanRange &table_scan_range, + ObMviewMergeWrapper *&merge_wrapper, + ObMviewMerge *&mview_merge); +private: + int get_mview_merge( + ObTableAccessParam ¶m, + ObTableAccessContext &context, + ObGetTableParam &get_table_param, + ObTableScanRange &table_scan_range, + ObMviewMerge *&version_merge); + ObMviewMerge *merges_[T_MAX_ITER_TYPE]; +}; + +} +} + +#endif diff --git a/src/storage/access/ob_multiple_scan_merge.cpp b/src/storage/access/ob_multiple_scan_merge.cpp index c71d5d57fc..298f56fe28 100644 --- a/src/storage/access/ob_multiple_scan_merge.cpp +++ b/src/storage/access/ob_multiple_scan_merge.cpp @@ -34,7 +34,6 @@ ObMultipleScanMerge::ObMultipleScanMerge() simple_merge_(nullptr), loser_tree_(nullptr), rows_merger_(nullptr), - iter_del_row_(false), consumer_cnt_(0), filt_del_count_(0), range_(NULL), @@ -308,7 +307,6 @@ void ObMultipleScanMerge::reset() loser_tree_ = nullptr; rows_merger_ = nullptr; tree_cmp_.reset(); - iter_del_row_ = false; consumer_cnt_ = 0; filt_del_count_ = 0; range_ = NULL; @@ -319,7 +317,6 @@ void ObMultipleScanMerge::reset() void ObMultipleScanMerge::reuse() { ObMultipleMerge::reuse(); - iter_del_row_ = false; consumer_cnt_ = 0; filt_del_count_ = 0; } @@ -328,7 +325,6 @@ void ObMultipleScanMerge::reclaim() { rows_merger_ = nullptr; tree_cmp_.reset(); - iter_del_row_ = false; consumer_cnt_ = 0; range_ = NULL; ObMultipleMerge::reclaim(); diff --git a/src/storage/access/ob_multiple_scan_merge.h b/src/storage/access/ob_multiple_scan_merge.h index 932480f253..82184230b3 100644 --- a/src/storage/access/ob_multiple_scan_merge.h +++ b/src/storage/access/ob_multiple_scan_merge.h @@ -44,7 +44,6 @@ public: virtual void reset() override; virtual void reuse() override; virtual void reclaim() override; - inline void set_iter_del_row(const bool iter_del_row) { iter_del_row_ = iter_del_row; } protected: virtual int calc_scan_range() override; virtual int construct_iters() override; @@ -64,7 +63,6 @@ protected: ObScanSimpleMerger *simple_merge_; ObScanMergeLoserTree *loser_tree_; common::ObRowsMerger *rows_merger_; - bool iter_del_row_; int64_t consumers_[common::MAX_TABLE_CNT_IN_STORAGE]; int64_t consumer_cnt_; private: diff --git a/src/storage/access/ob_pushdown_aggregate.cpp b/src/storage/access/ob_pushdown_aggregate.cpp index 230b9e86a2..c8bee5fac8 100644 --- a/src/storage/access/ob_pushdown_aggregate.cpp +++ b/src/storage/access/ob_pushdown_aggregate.cpp @@ -2918,7 +2918,8 @@ int ObPDAggFactory::alloc_cell( void *buf = nullptr; ObAggCell *cell = nullptr; switch (basic_info.agg_expr_->type_) { - case T_REF_COLUMN: { + case T_REF_COLUMN: + case T_ORA_ROWSCN: { if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObFirstRowAggCell))) || OB_ISNULL(cell = new(buf) ObFirstRowAggCell(basic_info, allocator_))) { ret = OB_ALLOCATE_MEMORY_FAILED; diff --git a/src/storage/access/ob_single_merge.cpp b/src/storage/access/ob_single_merge.cpp index ba69f108eb..bf92b4a071 100644 --- a/src/storage/access/ob_single_merge.cpp +++ b/src/storage/access/ob_single_merge.cpp @@ -57,7 +57,12 @@ int ObSingleMerge::open(const ObDatumRowkey &rowkey) STORAGE_LOG(WARN, "Failed to reserve full row", K(ret)); } if (OB_FAIL(ret)) { - } else if (OB_FAIL(fuse_row_cache_fetcher_.init(access_param_->iter_param_.tablet_id_, access_param_->iter_param_.get_read_info(), tablet_meta.clog_checkpoint_scn_.get_val_for_tx()))) { + } else if (OB_FAIL(fuse_row_cache_fetcher_.init(access_ctx_->get_scan_type(), + access_param_->iter_param_.tablet_id_, + access_param_->iter_param_.get_read_info(), + tablet_meta.clog_checkpoint_scn_.get_val_for_tx(), + access_ctx_->trans_version_range_.base_version_, + access_ctx_->trans_version_range_.snapshot_version_))) { STORAGE_LOG(WARN, "fail to init fuse row cache fetcher", K(ret)); } else { rowkey_ = &rowkey; @@ -221,9 +226,6 @@ int ObSingleMerge::get_and_fuse_cache_row(const int64_t read_snapshot_version, } else if (OB_FAIL(get_table_row(i, tables_, full_row_, final_result, have_uncommited_row))) { STORAGE_LOG(WARN, "fail to get table row", K(ret), K(i), K(full_row_), K(tables_)); } -#ifdef ENABLE_DEBUG_LOG - access_ctx_->defensive_check_record_.end_access_table_idx_ = i; -#endif } if (OB_SUCC(ret) && handle_.is_valid()) { ObDatumRow cache_row; @@ -236,9 +238,6 @@ int ObSingleMerge::get_and_fuse_cache_row(const int64_t read_snapshot_version, if (OB_FAIL(ObRowFuse::fuse_row(cache_row, fuse_row, nop_pos_, final_result))) { STORAGE_LOG(WARN, "fail to fuse row", K(ret)); } else { -#ifdef ENABLE_DEBUG_LOG - access_ctx_->defensive_check_record_.use_fuse_cache_data_ = true; -#endif STORAGE_LOG(TRACE, "fuse row cache", K(cache_row), K(fuse_row)); } } @@ -251,16 +250,16 @@ int ObSingleMerge::inner_get_next_row(ObDatumRow &row) { int ret = OB_SUCCESS; if (NULL != rowkey_ && 0 < tables_.count()) { - bool final_result = false; - int64_t table_idx = -1; ObITable *table = tables_.at(0); bool have_uncommited_row = false; + const StorageScanType scan_type = access_ctx_->get_scan_type(); const ObITableReadInfo *read_info = access_param_->iter_param_.get_read_info(); const ObTabletMeta &tablet_meta = get_table_param_->tablet_iter_.get_tablet()->get_tablet_meta(); const int64_t read_snapshot_version = access_ctx_->trans_version_range_.snapshot_version_; const bool enable_fuse_row_cache = access_ctx_->use_fuse_row_cache_ && - access_param_->iter_param_.enable_fuse_row_cache(access_ctx_->query_flag_) && - read_snapshot_version >= tablet_meta.snapshot_version_ && + access_param_->iter_param_.enable_fuse_row_cache(access_ctx_->query_flag_, scan_type) && + (is_mview_table_scan(scan_type) || + read_snapshot_version >= tablet_meta.snapshot_version_) && (!table->is_co_sstable() || static_cast(table)->is_all_cg_base()) && !tablet_meta.has_transfer_table(); // The query in the transfer scenario does not enable fuse row cache bool need_update_fuse_cache = false; @@ -273,55 +272,23 @@ int ObSingleMerge::inner_get_next_row(ObDatumRow &row) access_ctx_->use_fuse_row_cache_ = enable_fuse_row_cache; STORAGE_LOG(DEBUG, "single merge start to get next row", KPC(rowkey_), K(access_ctx_->use_fuse_row_cache_), - K(access_param_->iter_param_.enable_fuse_row_cache(access_ctx_->query_flag_)), K(access_param_->iter_param_)); - - for (table_idx = tables_.count() - 1; OB_SUCC(ret) && !final_result && table_idx >= 0; --table_idx) { - if (OB_ISNULL(table = tables_.at(table_idx))) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "Unexpected null table to single get", K(ret), K(table_idx), K(tables_)); - } else if (!table->is_memtable()) { - break; - } else if (OB_FAIL(get_table_row(table_idx, tables_, full_row_, final_result, have_uncommited_row))) { - STORAGE_LOG(WARN, "fail to get table row", K(ret), K(table_idx), K(full_row_), K(tables_)); - } else { -#ifdef ENABLE_DEBUG_LOG - if (table_idx == tables_.count() - 1) { - access_ctx_->defensive_check_record_.start_access_table_idx_ = table_idx; - access_ctx_->defensive_check_record_.total_table_handle_cnt_ = tables_.count(); - access_ctx_->defensive_check_record_.fist_access_table_start_scn_ = table->get_start_scn(); + K(access_param_->iter_param_.enable_fuse_row_cache(access_ctx_->query_flag_, scan_type)), K(access_param_->iter_param_)); + if (is_mview_table_scan(scan_type)) { + if (OB_FAIL(get_mview_table_scan_row(enable_fuse_row_cache, have_uncommited_row, need_update_fuse_cache))) { + STORAGE_LOG(WARN, "Failed to get mview table scan row", K(ret), K(enable_fuse_row_cache)); } -#endif - } - } - if (OB_FAIL(ret)) { - } else if (final_result) { -#ifdef ENABLE_DEBUG_LOG - access_ctx_->defensive_check_record_.is_all_data_from_memtable_ = true; -#endif - } else if (enable_fuse_row_cache) { - if (OB_FAIL(get_and_fuse_cache_row(read_snapshot_version, - tablet_meta.multi_version_start_, - full_row_, - final_result, - have_uncommited_row, - need_update_fuse_cache))) { - STORAGE_LOG(WARN, "Failed to get fuse cache row", K(ret), K(full_row_)); - } - } else { - // secondly, try to get from other delta table - for (; OB_SUCC(ret) && !final_result && table_idx >= 0; --table_idx) { - if (OB_FAIL(get_table_row(table_idx, tables_, full_row_, final_result, have_uncommited_row))) { - STORAGE_LOG(WARN, "fail to get table row", K(ret), K(table_idx), K(full_row_), K(tables_)); - } - } -#ifdef ENABLE_DEBUG_LOG - access_ctx_->defensive_check_record_.end_access_table_idx_ = table_idx; -#endif + } else if (OB_FAIL(get_normal_table_scan_row(read_snapshot_version, + tablet_meta.multi_version_start_, + enable_fuse_row_cache, + have_uncommited_row, + need_update_fuse_cache))) { + STORAGE_LOG(WARN, "Failed to get normal row", K(ret), K(read_snapshot_version), K(tablet_meta.multi_version_start_), + K(enable_fuse_row_cache)); } if (OB_SUCC(ret)) { - STORAGE_LOG(DEBUG, "row before project", K(full_row_)); - if (!full_row_.row_flag_.is_exist_without_delete()) { + STORAGE_LOG(DEBUG, "row before project", K(iter_del_row_), K(full_row_)); + if (!full_row_.row_flag_.is_exist_without_delete() && !(iter_del_row_ && full_row_.row_flag_.is_delete())) { ret = OB_ITER_END; } else { const ObColumnIndexArray &cols_index = read_info->get_columns_index(); @@ -337,10 +304,10 @@ int ObSingleMerge::inner_get_next_row(ObDatumRow &row) } if (OB_FAIL(ret)) { } else if (!have_uncommited_row && need_update_fuse_cache - && access_ctx_->enable_put_fuse_row_cache(SINGLE_GET_FUSE_ROW_CACHE_PUT_COUNT_THRESHOLD)) { + && access_ctx_->enable_put_fuse_row_cache(SINGLE_GET_FUSE_ROW_CACHE_PUT_COUNT_THRESHOLD, is_mview_table_scan(scan_type))) { // try to put row cache int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = fuse_row_cache_fetcher_.put_fuse_row_cache(*rowkey_, full_row_, read_snapshot_version))) { + if (OB_SUCCESS != (tmp_ret = fuse_row_cache_fetcher_.put_fuse_row_cache(*rowkey_, full_row_))) { STORAGE_LOG(WARN, "fail to put fuse row cache", K(ret), KPC(rowkey_), K(full_row_), K(read_snapshot_version)); } else { access_ctx_->table_store_stat_.fuse_row_cache_put_cnt_++; @@ -348,27 +315,6 @@ int ObSingleMerge::inner_get_next_row(ObDatumRow &row) } } } -#ifdef ENABLE_DEBUG_LOG - /* - if (OB_SUCC(ret)) { - access_ctx_->defensive_check_record_.query_flag_ = access_ctx_->query_flag_; - transaction::ObTransService *trx = MTL(transaction::ObTransService *); - bool trx_id_valid = (NULL != access_ctx_->store_ctx_ - && access_ctx_->store_ctx_->mvcc_acc_ctx_.snapshot_.tx_id_.is_valid()); - if (OB_NOT_NULL(trx) - && trx_id_valid - && NULL != trx->get_defensive_check_mgr()) { - (void)trx->get_defensive_check_mgr()->put(tablet_meta.tablet_id_, - access_ctx_->store_ctx_->mvcc_acc_ctx_.snapshot_.tx_id_, - row, - *rowkey_, - access_ctx_->defensive_check_record_); - } - } - access_ctx_->defensive_check_record_.reset(); - */ -#endif - // When the index lookups the rowkeys from the main table, it should exists // and if we find that it does not exist, there must be an anomaly if (GCONF.enable_defensive_check() @@ -389,5 +335,94 @@ int ObSingleMerge::inner_get_next_row(ObDatumRow &row) return ret; } +int ObSingleMerge::get_normal_table_scan_row(const int64_t read_snapshot_version, + const int64_t multi_version_start, + const bool enable_fuse_row_cache, + bool &have_uncommited_row, + bool &need_update_fuse_cache) +{ + int ret = OB_SUCCESS; + bool final_result = false; + int64_t table_idx = -1; + ObITable *table = nullptr; + for (table_idx = tables_.count() - 1; OB_SUCC(ret) && !final_result && table_idx >= 0; --table_idx) { + if (OB_ISNULL(table = tables_.at(table_idx))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Unexpected null table to single get", K(ret), K(table_idx), K(tables_)); + } else if (!table->is_memtable()) { + break; + } else if (OB_FAIL(get_table_row(table_idx, tables_, full_row_, final_result, have_uncommited_row))) { + STORAGE_LOG(WARN, "fail to get table row", K(ret), K(table_idx), K(full_row_), K(tables_)); + } + } + if (OB_FAIL(ret)) { + } else if (final_result) { + } else if (enable_fuse_row_cache) { + if (OB_FAIL(get_and_fuse_cache_row(read_snapshot_version, + multi_version_start, + full_row_, + final_result, + have_uncommited_row, + need_update_fuse_cache))) { + STORAGE_LOG(WARN, "Failed to get fuse cache row", K(ret), K(full_row_)); + } + } else { + // secondly, try to get from other delta table + for (; OB_SUCC(ret) && !final_result && table_idx >= 0; --table_idx) { + if (OB_FAIL(get_table_row(table_idx, tables_, full_row_, final_result, have_uncommited_row))) { + STORAGE_LOG(WARN, "fail to get table row", K(ret), K(table_idx), K(full_row_), K(tables_)); + } + } + } + return ret; +} + +int ObSingleMerge::get_mview_table_scan_row(const bool enable_fuse_row_cache, + bool &have_uncommited_row, + bool &need_update_fuse_cache) +{ + int ret = OB_SUCCESS; + bool final_result = false; + if (enable_fuse_row_cache) { + if (OB_FAIL(fuse_row_cache_fetcher_.get_fuse_row_cache(*rowkey_, handle_))) { + if (OB_ENTRY_NOT_EXIST != ret) { + STORAGE_LOG(WARN, "Failed to get from fuse row cache", K(ret), KPC(rowkey_)); + } else { + ++access_ctx_->table_store_stat_.fuse_row_cache_miss_cnt_; + ret = OB_SUCCESS; + } + } else if (handle_.is_valid()) { + ObDatumRow cache_row; + cache_row.count_ = handle_.value_->get_column_cnt(); + cache_row.storage_datums_ = handle_.value_->get_datums(); + cache_row.row_flag_ = handle_.value_->get_flag(); + ++access_ctx_->table_store_stat_.fuse_row_cache_hit_cnt_; + STORAGE_LOG(DEBUG, "find fuse row cache", K(handle_), KPC(rowkey_)); + if (cache_row.row_flag_.is_exist()) { + if (OB_FAIL(ObRowFuse::fuse_row(cache_row, full_row_, nop_pos_, final_result))) { + STORAGE_LOG(WARN, "Failed to fuse row", K(ret)); + } else { + STORAGE_LOG(TRACE, "fuse row cache", K(cache_row), K(full_row_), K(final_result)); + final_result = true; + } + } + } + } + if (OB_SUCC(ret) && !final_result) { + need_update_fuse_cache = enable_fuse_row_cache; + int64_t table_idx = -1; + ObITable *table = nullptr; + for (table_idx = tables_.count() - 1; OB_SUCC(ret) && !final_result && table_idx >= 0; --table_idx) { + if (OB_ISNULL(table = tables_.at(table_idx))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Unexpected null table to single get", K(ret), K(table_idx), K(tables_)); + } else if (OB_FAIL(get_table_row(table_idx, tables_, full_row_, final_result, have_uncommited_row))) { + STORAGE_LOG(WARN, "Failed to get table row", K(ret), K(table_idx), K(full_row_), K(tables_)); + } + } + } + return ret; +} + } /* namespace storage */ } /* namespace oceanbase */ diff --git a/src/storage/access/ob_single_merge.h b/src/storage/access/ob_single_merge.h index 11acd35075..8ea582f125 100644 --- a/src/storage/access/ob_single_merge.h +++ b/src/storage/access/ob_single_merge.h @@ -46,6 +46,14 @@ private: bool &final_result, bool &have_uncommited_row, bool &need_update_fuse_cache); + int get_normal_table_scan_row(const int64_t read_snapshot_version, + const int64_t multi_version_start, + const bool enable_fuse_row_cache, + bool &have_uncommited_row, + bool &need_update_fuse_cache); + int get_mview_table_scan_row(const bool enable_fuse_row_cache, + bool &have_uncommited_row, + bool &need_update_fuse_cache); private: static const int64_t SINGLE_GET_FUSE_ROW_CACHE_PUT_COUNT_THRESHOLD = 50; const blocksstable::ObDatumRowkey *rowkey_; diff --git a/src/storage/access/ob_sstable_row_getter.cpp b/src/storage/access/ob_sstable_row_getter.cpp index 72a278fe87..4ec4f88f7f 100644 --- a/src/storage/access/ob_sstable_row_getter.cpp +++ b/src/storage/access/ob_sstable_row_getter.cpp @@ -125,7 +125,7 @@ int ObSSTableRowGetter::inner_get_next_row(const ObDatumRow *&store_row) ObDatumRow &datum_row = *const_cast(store_row); if (!store_row->row_flag_.is_not_exist() && iter_param_->need_scn_ && - OB_FAIL(set_row_scn(*iter_param_, store_row))) { + OB_FAIL(set_row_scn(access_ctx_->use_fuse_row_cache_, *iter_param_, store_row))) { LOG_WARN("failed to set row scn", K(ret)); } EVENT_INC(ObStatEventIds::SSSTORE_READ_ROW_COUNT); diff --git a/src/storage/access/ob_sstable_row_multi_getter.cpp b/src/storage/access/ob_sstable_row_multi_getter.cpp index a2efc24a08..76733f14c7 100644 --- a/src/storage/access/ob_sstable_row_multi_getter.cpp +++ b/src/storage/access/ob_sstable_row_multi_getter.cpp @@ -129,7 +129,7 @@ int ObSSTableRowMultiGetter::inner_get_next_row(const blocksstable::ObDatumRow * ObDatumRow &datum_row = *const_cast(store_row); if (!store_row->row_flag_.is_not_exist() && iter_param_->need_scn_ && - OB_FAIL(set_row_scn(*iter_param_, store_row))) { + OB_FAIL(set_row_scn(access_ctx_->use_fuse_row_cache_, *iter_param_, store_row))) { LOG_WARN("failed to set row scn", K(ret)); } EVENT_INC(ObStatEventIds::SSSTORE_READ_ROW_COUNT); diff --git a/src/storage/access/ob_sstable_row_scanner.cpp b/src/storage/access/ob_sstable_row_scanner.cpp index 099e1222d2..30c08c4953 100644 --- a/src/storage/access/ob_sstable_row_scanner.cpp +++ b/src/storage/access/ob_sstable_row_scanner.cpp @@ -93,6 +93,7 @@ bool ObSSTableRowScanner::can_batch_scan() const { return can_blockscan() && block_row_store_->filter_applied() && + !access_ctx_->is_mview_query() && // can batch scan when only enable_pd_aggregate, as it uses own datum buffer and only return aggregated result (iter_param_->vectorized_enabled_ || iter_param_->enable_pd_aggregate()); } @@ -338,7 +339,7 @@ int ObSSTableRowScanner::inner_get_next_row(const ObDatumRow *&sto ObDatumRow &datum_row = *const_cast(store_row); if (!store_row->row_flag_.is_not_exist() && iter_param_->need_scn_ && - OB_FAIL(set_row_scn(*iter_param_, store_row))) { + OB_FAIL(set_row_scn(access_ctx_->use_fuse_row_cache_, *iter_param_, store_row))) { LOG_WARN("failed to set row scn", K(ret)); } EVENT_INC(ObStatEventIds::SSSTORE_READ_ROW_COUNT); diff --git a/src/storage/access/ob_table_access_context.cpp b/src/storage/access/ob_table_access_context.cpp index 4e68b9c90a..ea22112b1e 100644 --- a/src/storage/access/ob_table_access_context.cpp +++ b/src/storage/access/ob_table_access_context.cpp @@ -55,6 +55,7 @@ ObTableAccessContext::ObTableAccessContext() : is_inited_(false), use_fuse_row_cache_(false), need_scn_(false), + need_release_mview_scan_info_(true), timeout_(0), query_flag_(), sql_mode_(0), @@ -79,7 +80,8 @@ ObTableAccessContext::ObTableAccessContext() cg_param_pool_(nullptr), block_row_store_(nullptr), sample_filter_(nullptr), - trans_state_mgr_(nullptr) + trans_state_mgr_(nullptr), + mview_scan_info_(nullptr) { merge_scn_.set_max(); } @@ -103,6 +105,12 @@ ObTableAccessContext::~ObTableAccessContext() cg_iter_pool_ = nullptr; } ObRowSampleFilterFactory::destroy_sample_filter(sample_filter_); + if (need_release_mview_scan_info_) { + release_mview_scan_info(stmt_allocator_, mview_scan_info_); + need_release_mview_scan_info_ = false; + } else { + mview_scan_info_ = nullptr; + } } int ObTableAccessContext::build_lob_locator_helper(ObTableScanParam &scan_param, @@ -301,6 +309,46 @@ int ObTableAccessContext::init(const common::ObQueryFlag &query_flag, return ret; } +int ObTableAccessContext::init_for_mview(common::ObIAllocator *allocator, const ObTableAccessContext &access_ctx, ObStoreCtx &store_ctx) +{ + int ret = OB_SUCCESS; + if (is_inited_ && stmt_allocator_ != access_ctx.stmt_allocator_) { + ret = OB_INIT_TWICE; + LOG_WARN("cannot init twice", K(ret)); + } else { + stmt_allocator_ = access_ctx.stmt_allocator_; + allocator_ = allocator; + cached_iter_node_ = nullptr; + range_allocator_ = nullptr; + ls_id_ = access_ctx.ls_id_; + tablet_id_ = access_ctx.tablet_id_; + query_flag_ = access_ctx.query_flag_; + sql_mode_ = access_ctx.sql_mode_; + timeout_ = access_ctx.timeout_; + store_ctx_ = &store_ctx; + table_scan_stat_ = nullptr; + limit_param_ = nullptr; + trans_version_range_.base_version_ = 0; + trans_version_range_.snapshot_version_ = access_ctx.trans_version_range_.base_version_; + need_scn_ = false; + range_array_pos_ = nullptr; + use_fuse_row_cache_ = true; + lob_locator_helper_ = nullptr; + if (!micro_block_handle_mgr_.is_valid() && + OB_FAIL(micro_block_handle_mgr_.init( + false, + table_store_stat_, + query_flag_))) { + LOG_WARN("Failed to init micro block handle mgr", K(ret)); + } else { + mview_scan_info_ = access_ctx.mview_scan_info_; + need_release_mview_scan_info_ = false; + is_inited_ = true; + } + } + return ret; +} + void ObTableAccessContext::inc_micro_access_cnt() { ++table_store_stat_.micro_access_cnt_; @@ -330,9 +378,29 @@ int ObTableAccessContext::init_scan_allocator(ObTableScanParam &scan_param) return ret; } +int ObTableAccessContext::init_mview_scan_info(const int64_t multi_version_start, const sql::ObExprPtrIArray *op_filters, sql::ObEvalCtx &eval_ctx) +{ + int ret = OB_SUCCESS; + if (nullptr != mview_scan_info_) { + } else if (OB_FAIL(build_mview_scan_info_if_need(query_flag_, op_filters, eval_ctx, stmt_allocator_, mview_scan_info_))) { + LOG_WARN("Failed to build mview scan info", K(ret)); + } + if (OB_FAIL(ret) || nullptr == mview_scan_info_) { + } else if (OB_FAIL(mview_scan_info_->check_and_update_version_range(multi_version_start, trans_version_range_))) { + LOG_WARN("Failed to check and update version range", K(ret), K(multi_version_start), K(trans_version_range_), KPC_(mview_scan_info)); + } + return ret; +} + void ObTableAccessContext::reset() { reset_lob_locator_helper(); + if (need_release_mview_scan_info_) { + release_mview_scan_info(stmt_allocator_, mview_scan_info_); + need_release_mview_scan_info_ = false; + } else { + mview_scan_info_ = nullptr; + } cached_iter_node_ = nullptr; if (nullptr != stmt_iter_pool_) { stmt_iter_pool_->~ObStoreRowIterPool(); diff --git a/src/storage/access/ob_table_access_context.h b/src/storage/access/ob_table_access_context.h index fdd91defff..26686be48a 100644 --- a/src/storage/access/ob_table_access_context.h +++ b/src/storage/access/ob_table_access_context.h @@ -37,6 +37,7 @@ template class ObStoreRowIterPool; class ObBlockRowStore; class ObCGIterParamPool; +struct ObTableScanRange; #define REALTIME_MONITOR_ADD_IO_READ_BYTES(CTX, SIZE) \ if (OB_NOT_NULL(CTX)) CTX->add_io_read_bytes(SIZE) \ @@ -147,8 +148,8 @@ struct ObTableAccessContext inline bool enable_get_fuse_row_cache(const int64_t threshold) const { return query_flag_.is_use_fuse_row_cache() && table_store_stat_.enable_get_fuse_row_cache(threshold) && !need_scn_ && !tablet_id_.is_ls_inner_tablet(); } - inline bool enable_put_fuse_row_cache(const int64_t threshold) const { - return query_flag_.is_use_fuse_row_cache() && table_store_stat_.enable_put_fuse_row_cache(threshold) && !need_scn_ && !tablet_id_.is_ls_inner_tablet(); + inline bool enable_put_fuse_row_cache(const int64_t threshold, const bool is_mview_table_scan) const { + return query_flag_.is_use_fuse_row_cache() && table_store_stat_.enable_put_fuse_row_cache(threshold) && (!need_scn_ || is_mview_table_scan) && !tablet_id_.is_ls_inner_tablet(); } inline bool is_limit_end() const { return (nullptr != limit_param_ && limit_param_->limit_ >= 0 && (out_cnt_ - limit_param_->offset_ >= limit_param_->limit_)); @@ -193,9 +194,22 @@ struct ObTableAccessContext common::ObIAllocator &allocator, const common::ObVersionRange &trans_version_range, CachedIteratorNode *cached_iter_node = nullptr); + // used for mview table scan + int init_for_mview(common::ObIAllocator *allocator, + const ObTableAccessContext &access_ctx, + ObStoreCtx &store_ctx); + OB_INLINE bool is_mview_query() const + { + return nullptr != mview_scan_info_; + } + OB_INLINE StorageScanType get_scan_type() const + { + return is_mview_query() ? mview_scan_info_->scan_type_ : StorageScanType::NORMAL; + } int alloc_iter_pool(const bool use_column_store); void inc_micro_access_cnt(); int init_scan_allocator(ObTableScanParam &scan_param); + int init_mview_scan_info(const int64_t multi_version_start, const sql::ObExprPtrIArray *op_filters, sql::ObEvalCtx &eval_ctx); // update realtime monitor info OB_INLINE void add_io_read_bytes(const int64_t bytes) { @@ -225,6 +239,7 @@ struct ObTableAccessContext K_(is_inited), K_(use_fuse_row_cache), K_(need_scn), + K_(need_release_mview_scan_info), K_(timeout), K_(ls_id), K_(tablet_id), @@ -246,7 +261,8 @@ struct ObTableAccessContext KP_(cg_iter_pool), KP_(cg_param_pool), KP_(block_row_store), - KP_(sample_filter)); + KP_(sample_filter), + KPC_(mview_scan_info)); private: static const int64_t DEFAULT_COLUMN_SCALE_INFO_SIZE = 8; static const int64_t USE_BLOCK_CACHE_LIMIT = 128L << 10; // 128K @@ -257,7 +273,6 @@ private: const ObVersionRange &trans_version_range); // local scan // init need_fill_scale_ and search column which need fill scale int init_column_scale_info(ObTableScanParam &scan_param); - public: OB_INLINE common::ObIAllocator *get_long_life_allocator() { @@ -270,6 +285,7 @@ public: bool is_inited_; bool use_fuse_row_cache_; // temporary code bool need_scn_; + bool need_release_mview_scan_info_; int64_t timeout_; share::ObLSID ls_id_; common::ObTabletID tablet_id_; @@ -300,9 +316,7 @@ public: ObBlockRowStore *block_row_store_; ObRowSampleFilter *sample_filter_; compaction::ObCachedTransStateMgr *trans_state_mgr_; -#ifdef ENABLE_DEBUG_LOG - transaction::ObDefensiveCheckRecordExtend defensive_check_record_; -#endif + ObMviewScanInfo *mview_scan_info_; }; } // namespace storage diff --git a/src/storage/access/ob_table_access_param.cpp b/src/storage/access/ob_table_access_param.cpp index d01e61434f..fd3ee14c7a 100644 --- a/src/storage/access/ob_table_access_param.cpp +++ b/src/storage/access/ob_table_access_param.cpp @@ -140,11 +140,11 @@ int ObTableIterParam::refresh_lob_column_out_status() return ret; } -bool ObTableIterParam::enable_fuse_row_cache(const ObQueryFlag &query_flag) const +bool ObTableIterParam::enable_fuse_row_cache(const ObQueryFlag &query_flag, const StorageScanType scan_type) const { bool bret = is_x86() && query_flag.is_use_fuse_row_cache() && !query_flag.is_read_latest() && - nullptr != rowkey_read_info_ && !need_scn_ && is_same_schema_column_ && - !has_virtual_columns_ && !has_lob_column_out_; + nullptr != rowkey_read_info_ && (!need_scn_ || is_mview_table_scan(scan_type)) && + is_same_schema_column_ && !has_virtual_columns_ && !has_lob_column_out_; return bret; } @@ -339,6 +339,7 @@ int ObTableAccessParam::init( iter_param_.vectorized_enabled_ = nullptr != get_op() && get_op()->is_vectorized(); iter_param_.limit_prefetch_ = (nullptr == op_filters_ || op_filters_->empty()); iter_param_.is_mds_query_ = scan_param.is_mds_query_; + if (iter_param_.is_use_column_store() && nullptr != table_param.get_read_info().get_cg_idxs() && !iter_param_.need_fill_group_idx()) { // not use column store in group rescan @@ -347,7 +348,8 @@ int ObTableAccessParam::init( iter_param_.set_not_use_column_store(); } if (scan_param.need_switch_param_ || - iter_param_.is_use_column_store()) { + iter_param_.is_use_column_store() || + scan_param.is_mview_query()) { iter_param_.set_use_stmt_iter_pool(); } @@ -460,12 +462,13 @@ DEF_TO_STRING(ObTableAccessParam) } int set_row_scn( + const bool use_fuse_row_cache, const ObTableIterParam &iter_param, const ObDatumRow *store_row) { int ret = OB_SUCCESS; const ObColDescIArray *out_cols = nullptr; - const ObITableReadInfo *read_info = iter_param.get_read_info(); + const ObITableReadInfo *read_info = iter_param.get_read_info(use_fuse_row_cache); if (OB_UNLIKELY(nullptr == read_info)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected null read info", K(ret)); diff --git a/src/storage/access/ob_table_access_param.h b/src/storage/access/ob_table_access_param.h index b468f682c2..160c6b6a04 100644 --- a/src/storage/access/ob_table_access_param.h +++ b/src/storage/access/ob_table_access_param.h @@ -49,7 +49,7 @@ public: void reset(); bool is_valid() const; int refresh_lob_column_out_status(); - bool enable_fuse_row_cache(const ObQueryFlag &query_flag) const; + bool enable_fuse_row_cache(const ObQueryFlag &query_flag, const StorageScanType scan_type) const; //temp solution int get_cg_column_param(const share::schema::ObColumnParam *&column_param) const; int build_index_filter_for_row_store(common::ObIAllocator *allocator); @@ -111,6 +111,10 @@ public: { return (read_info_ != nullptr && read_info_ != rowkey_read_info_) ? read_info_->get_group_idx_col_index() : common::OB_INVALID_INDEX; } + OB_INLINE int64_t get_mview_old_new_col_index() const + { + return (read_info_ != nullptr && read_info_ != rowkey_read_info_) ? read_info_->get_mview_old_new_col_index() : common::OB_INVALID_INDEX; + } bool can_be_reused(const uint32_t cg_idx, const common::ObIArray &exprs, const bool is_aggregate) { const sql::ObExprPtrIArray *inner_exprs = is_aggregate ? aggregate_exprs_ : output_exprs_; @@ -285,6 +289,7 @@ public: //TODO @hanhui remove this func int set_row_scn( + const bool use_fuse_row_cache, const ObTableIterParam &iter_param, const blocksstable::ObDatumRow *store_row); diff --git a/src/storage/access/ob_table_read_info.cpp b/src/storage/access/ob_table_read_info.cpp index 61c9646b2b..6f481b1efd 100644 --- a/src/storage/access/ob_table_read_info.cpp +++ b/src/storage/access/ob_table_read_info.cpp @@ -226,7 +226,7 @@ void ObReadInfoStruct::reset() is_oracle_mode_ = false; allocator_ = nullptr; schema_column_count_ = 0; - compat_version_ = READ_INFO_VERSION_V2; + compat_version_ = READ_INFO_VERSION_V3; reserved_ = 0; schema_rowkey_cnt_ = 0; rowkey_cnt_ = 0; @@ -302,8 +302,10 @@ int ObReadInfoStruct::init_compat_version() LOG_WARN("fail to get data version", K(ret)); } else if (compat_version < DATA_VERSION_4_3_0_0) { compat_version_ = READ_INFO_VERSION_V1; - } else { + } else if (compat_version < DATA_VERSION_4_3_4_0) { compat_version_ = READ_INFO_VERSION_V2; + } else { + compat_version_ = READ_INFO_VERSION_V3; } return ret; } @@ -315,6 +317,7 @@ ObTableReadInfo::ObTableReadInfo() : ObReadInfoStruct(false/*rowkey_mode*/), trans_col_index_(OB_INVALID_INDEX), group_idx_col_index_(OB_INVALID_INDEX), + mview_old_new_col_index_(OB_INVALID_INDEX), seq_read_column_count_(0), max_col_index_(-1), cols_param_(), @@ -467,6 +470,9 @@ void ObTableReadInfo::inner_gene_cols_index_by_col_descs( } else if (common::OB_HIDDEN_GROUP_IDX_COLUMN_ID == cols_desc.at(i).col_id_) { group_idx_col_index_ = i; col_index = -1; + } else if (common::OB_MAJOR_REFRESH_MVIEW_OLD_NEW_COLUMN_ID == cols_desc.at(i).col_id_) { + mview_old_new_col_index_ = i; + col_index = -1; } else { col_index = -1; } @@ -526,6 +532,7 @@ void ObTableReadInfo::reset() ObReadInfoStruct::reset(); trans_col_index_ = OB_INVALID_INDEX; group_idx_col_index_ = OB_INVALID_INDEX; + mview_old_new_col_index_ = OB_INVALID_INDEX; seq_read_column_count_ = 0; max_col_index_ = -1; cols_param_.reset(); @@ -575,6 +582,11 @@ int ObTableReadInfo::serialize( if (OB_SUCC(ret) && compat_version_ >= READ_INFO_VERSION_V2) { LST_DO_CODE(OB_UNIS_ENCODE, cg_idxs_, cols_extend_, has_all_column_group_); } + if (OB_SUCC(ret) && compat_version_ >= READ_INFO_VERSION_V3) { + if (OB_FAIL(serialization::encode_vi64(buf, buf_len, pos, mview_old_new_col_index_))) { + LOG_WARN("Fail to encode mview old new col index", K(ret)); + } + } return ret; } @@ -652,6 +664,15 @@ int ObTableReadInfo::deserialize( has_all_column_group_ = true; } } + if (OB_SUCC(ret)) { + if (compat_version_ >= READ_INFO_VERSION_V3) { + if (OB_FAIL(serialization::decode_vi64(buf, data_len, pos, &mview_old_new_col_index_))) { + LOG_WARN("Fail to decode mview old new col index", K(ret)); + } + } else { + mview_old_new_col_index_ = OB_INVALID_INDEX; + } + } if (OB_SUCC(ret) && cols_desc_.count() > 0) { const bool is_cg_sstable = ObCGReadInfo::is_cg_sstable(schema_rowkey_cnt_, schema_column_count_); @@ -709,6 +730,9 @@ int64_t ObTableReadInfo::get_serialize_size() const if (OB_SUCC(ret) && compat_version_ >= READ_INFO_VERSION_V2) { LST_DO_CODE(OB_UNIS_ADD_LEN, cg_idxs_, cols_extend_, has_all_column_group_); } + if (OB_SUCC(ret) && compat_version_ >= READ_INFO_VERSION_V3) { + len += serialization::encoded_length_vi64(mview_old_new_col_index_); + } return len; } @@ -722,6 +746,8 @@ int64_t ObTableReadInfo::to_string(char *buf, const int64_t buf_len) const K_(schema_rowkey_cnt), K_(rowkey_cnt), K_(trans_col_index), + K_(mview_old_new_col_index), + K_(group_idx_col_index), K_(seq_read_column_count), K_(max_col_index), K_(is_oracle_mode), diff --git a/src/storage/access/ob_table_read_info.h b/src/storage/access/ob_table_read_info.h index 5688a472b7..f27b003b1f 100644 --- a/src/storage/access/ob_table_read_info.h +++ b/src/storage/access/ob_table_read_info.h @@ -122,6 +122,7 @@ public: virtual int64_t get_schema_rowkey_count() const = 0; virtual int64_t get_rowkey_count() const = 0; virtual int64_t get_group_idx_col_index() const = 0; + virtual int64_t get_mview_old_new_col_index() const = 0; virtual int64_t get_max_col_index() const = 0; virtual int64_t get_trans_col_index() const = 0; virtual const common::ObIArray &get_columns_desc() const = 0; @@ -147,7 +148,7 @@ public: is_oracle_mode_(false), allocator_(nullptr), schema_column_count_(0), - compat_version_(READ_INFO_VERSION_V2), + compat_version_(READ_INFO_VERSION_V3), reserved_(0), schema_rowkey_cnt_(0), rowkey_cnt_(0), @@ -185,6 +186,10 @@ public: { return OB_INVALID_INDEX; } + OB_INLINE virtual int64_t get_mview_old_new_col_index() const override + { + return OB_INVALID_INDEX; + } OB_INLINE virtual int64_t get_max_col_index() const override { OB_ASSERT_MSG(false, "ObReadInfoStruct dose not promise max col index"); @@ -238,9 +243,10 @@ public: const int64_t col_cnt); int init_compat_version(); protected: - const int64_t READ_INFO_VERSION_V0 = 0; - const int64_t READ_INFO_VERSION_V1 = 1; - const int64_t READ_INFO_VERSION_V2 = 2; + static const int64_t READ_INFO_VERSION_V0 = 0; + static const int64_t READ_INFO_VERSION_V1 = 1; + static const int64_t READ_INFO_VERSION_V2 = 2; + static const int64_t READ_INFO_VERSION_V3 = 3; bool is_inited_; bool is_oracle_mode_; ObIAllocator *allocator_; @@ -307,6 +313,8 @@ public: { return trans_col_index_; } OB_INLINE int64_t get_group_idx_col_index() const { return group_idx_col_index_; } + OB_INLINE int64_t get_mview_old_new_col_index() const + { return mview_old_new_col_index_; } OB_INLINE int64_t get_seq_read_column_count() const { return seq_read_column_count_; } virtual const common::ObIArray *get_columns() const @@ -364,6 +372,7 @@ private: // distinguish schema changed by schema column count int64_t trans_col_index_; int64_t group_idx_col_index_; + int64_t mview_old_new_col_index_; // the count of common prefix between request columns and store columns int64_t seq_read_column_count_; int64_t max_col_index_; @@ -466,6 +475,10 @@ public: { return OB_INVALID_INDEX; } + OB_INLINE virtual int64_t get_mview_old_new_col_index() const override + { + return OB_INVALID_INDEX; + } virtual int64_t get_max_col_index() const { OB_ASSERT_MSG(false, "ObCGReadInfo dose not promise max col index"); @@ -525,6 +538,10 @@ public: { return OB_INVALID_INDEX; } + OB_INLINE virtual int64_t get_mview_old_new_col_index() const override + { + return OB_INVALID_INDEX; + } virtual int64_t get_max_col_index() const override { OB_ASSERT_MSG(false, "ObCGRowkeyReadInfo dose not promise max col index"); diff --git a/src/storage/access/ob_table_scan_iterator.cpp b/src/storage/access/ob_table_scan_iterator.cpp index 821aba638d..5a7d6e6155 100644 --- a/src/storage/access/ob_table_scan_iterator.cpp +++ b/src/storage/access/ob_table_scan_iterator.cpp @@ -54,6 +54,7 @@ ObTableScanIterator::ObTableScanIterator() row_sample_iterator_(NULL), block_sample_iterator_(NULL), // i_sample_iter_(NULL), + mview_merge_wrapper_(NULL), main_table_param_(), main_table_ctx_(), get_table_param_(), @@ -82,6 +83,7 @@ void ObTableScanIterator::reset() reset_scan_iter(skip_scan_merge_); reset_scan_iter(memtable_row_sample_iterator_); reset_scan_iter(block_sample_iterator_); + reset_scan_iter(mview_merge_wrapper_); // reset_scan_iter(i_sample_iter_); if (nullptr != cached_iter_node_) { ObGlobalIteratorPool *iter_pool = MTL(ObGlobalIteratorPool*); @@ -128,6 +130,7 @@ void ObTableScanIterator::reuse_row_iters() REUSE_SCAN_ITER(memtable_row_sample_iterator_); REUSE_SCAN_ITER(block_sample_iterator_); // REUSE_SCAN_ITER(i_sample_iter_); + REUSE_SCAN_ITER(mview_merge_wrapper_); #undef REUSE_SCAN_ITER } @@ -161,7 +164,8 @@ bool ObTableScanIterator::can_use_global_iter_pool(const ObQRIterType iter_type) main_table_param_.iter_param_.enable_pd_aggregate() || main_table_param_.iter_param_.enable_pd_group_by() || main_table_param_.iter_param_.is_column_replica_table_ || - main_table_param_.iter_param_.has_lob_column_out_) { + main_table_param_.iter_param_.has_lob_column_out_ || + scan_param_->is_mview_query()) { } else { const int64_t table_cnt = get_table_param_.tablet_iter_.table_iter()->count(); const int64_t col_cnt = MAX(scan_param_->table_param_->get_read_info().get_schema_column_count(), @@ -232,12 +236,24 @@ int ObTableScanIterator::prepare_table_context() STORAGE_LOG(WARN, "trans version range is not valid", K(ret), K(trans_version_range)); } else if (OB_FAIL(main_table_ctx_.init(*scan_param_, ctx_guard_.get_store_ctx(), trans_version_range, cached_iter_node_))) { STORAGE_LOG(WARN, "failed to init main table ctx", K(ret)); + } else if (scan_param_->is_mview_query()) { + const ObTabletMeta &tablet_meta = get_table_param_.tablet_iter_.get_tablet()->get_tablet_meta(); + if (OB_ISNULL(main_table_param_.op_filters_) || scan_param_->table_param_->use_lob_locator()) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Unexpected null scn filter or use lob locator in mview query", K(ret), KP(main_table_param_.op_filters_), + K(scan_param_->table_param_->use_lob_locator())); + } else if (OB_FAIL(main_table_ctx_.init_mview_scan_info(tablet_meta.multi_version_start_, + main_table_param_.op_filters_, + main_table_param_.get_op()->get_eval_ctx()))) { + STORAGE_LOG(WARN, "failed to init mview scan info", K(ret)); + } } } return ret; } -int ObTableScanIterator::switch_scan_param(ObMultipleMerge &iter) +template +int ObTableScanIterator::switch_scan_param(T &iter) { int ret = OB_SUCCESS; if (OB_FAIL(iter.switch_param(main_table_param_, main_table_ctx_, get_table_param_))) { @@ -406,6 +422,7 @@ int ObTableScanIterator::rescan_for_iter() RESET_NOT_REFRESHED_ITER(get_table_param_.refreshed_merge_, skip_scan_merge_); RESET_NOT_REFRESHED_ITER(get_table_param_.refreshed_merge_, memtable_row_sample_iterator_); RESET_NOT_REFRESHED_ITER(get_table_param_.refreshed_merge_, block_sample_iterator_); + RESET_NOT_REFRESHED_ITER(get_table_param_.refreshed_merge_, mview_merge_wrapper_); get_table_param_.refreshed_merge_ = nullptr; } #undef RESET_NOT_REFRESHED_ITER @@ -430,6 +447,11 @@ int ObTableScanIterator::switch_param_for_iter() SWITCH_PARAM_FOR_ITER(multi_scan_merge_, ret); SWITCH_PARAM_FOR_ITER(skip_scan_merge_, ret); #undef SWITCH_PARAM_FOR_ITER + if (OB_SUCC(ret) && nullptr != mview_merge_wrapper_) { + if (OB_FAIL(mview_merge_wrapper_->switch_param(main_table_param_, main_table_ctx_, get_table_param_))) { + STORAGE_LOG(WARN, "Failed to switch param", K(ret)); + } + } return ret; } @@ -519,7 +541,15 @@ int ObTableScanIterator::open_iter() } else { get_table_param_.frozen_version_ = scan_param_->frozen_version_; get_table_param_.sample_info_ = scan_param_->sample_info_; - if (table_scan_range_.is_get()) { + if (main_table_ctx_.is_mview_query()) { + ObMviewMerge *mview_merge = nullptr; + if (OB_FAIL(ObMviewMergeWrapper::alloc_mview_merge(main_table_param_, main_table_ctx_, get_table_param_, + table_scan_range_, mview_merge_wrapper_, mview_merge))) { + STORAGE_LOG(WARN, "Failed to alloc mview merge", K(ret)); + } else { + main_iter_ = mview_merge; + } + } else if (table_scan_range_.is_get()) { if (OB_FAIL(init_and_open_get_merge_iter_())) { STORAGE_LOG(WARN, "init and open get merge iterator failed", KR(ret)); } diff --git a/src/storage/access/ob_table_scan_iterator.h b/src/storage/access/ob_table_scan_iterator.h index 3a6a752ae3..995df175a5 100644 --- a/src/storage/access/ob_table_scan_iterator.h +++ b/src/storage/access/ob_table_scan_iterator.h @@ -28,6 +28,7 @@ #include "ob_multiple_skip_scan_merge.h" #include "ob_multiple_multi_skip_scan_merge.h" #include "ob_single_merge.h" +#include "ob_multiple_mview_merge.h" #include "storage/tx_storage/ob_access_service.h" #include "storage/tx_storage/ob_ls_map.h" #include "storage/tx/ob_trans_service.h" @@ -77,7 +78,7 @@ private: void try_release_cached_iter_node(const ObQRIterType rescan_iter_type); template int init_scan_iter(T *&iter); template void reset_scan_iter(T *&iter); - int switch_scan_param(ObMultipleMerge &iter); + template int switch_scan_param(T &iter); void reuse_row_iters(); int rescan_for_iter(); int switch_param_for_iter(); @@ -109,6 +110,7 @@ private: ObMemtableRowSampleIterator *memtable_row_sample_iterator_; ObRowSampleIterator *row_sample_iterator_; ObBlockSampleIterator *block_sample_iterator_; // TODO: @yuanzhe refactor + ObMviewMergeWrapper *mview_merge_wrapper_; // we should consider the constructor cost ObTableAccessParam main_table_param_; ObTableAccessContext main_table_ctx_; diff --git a/src/storage/backup/ob_backup_ctx.cpp b/src/storage/backup/ob_backup_ctx.cpp index cee760ed32..cfee2085c0 100644 --- a/src/storage/backup/ob_backup_ctx.cpp +++ b/src/storage/backup/ob_backup_ctx.cpp @@ -25,6 +25,7 @@ #include "storage/blocksstable/ob_logic_macro_id.h" #include "share/backup/ob_backup_data_table_operator.h" #include "common/storage/ob_device_common.h" +#include "storage/backup/ob_backup_data_store.h" #include @@ -1233,7 +1234,9 @@ ObLSBackupCtx::ObLSBackupCtx() check_tablet_info_cost_time_(), backup_tx_table_filled_tx_scn_(share::SCN::min_scn()), index_builder_mgr_(), - bandwidth_throttle_(NULL) + bandwidth_throttle_(NULL), + mview_dep_tablet_set_(), + wait_reuse_across_sstable_time_(0) {} ObLSBackupCtx::~ObLSBackupCtx() @@ -1275,6 +1278,8 @@ int ObLSBackupCtx::open( LOG_WARN("failed to init index builder mg", K(ret), K(param)); } else if (OB_FAIL(param_.assign(param))) { LOG_WARN("failed to assign param", K(ret), K(param)); + } else if (OB_FAIL(mview_dep_tablet_set_.create(DEFAULT_MVIEW_DEP_TABLET_SET_SIZE))) { + LOG_WARN("failed to create mveiw dep tablet set", K(ret)); } else { max_file_id_ = 0; prefetch_task_id_ = 0; @@ -1289,6 +1294,10 @@ int ObLSBackupCtx::open( LOG_WARN("failed to prepare tablet id reader", K(ret), K(param)); } else if (OB_FAIL(get_all_tablet_id_list_(reader, tablet_list))) { LOG_WARN("failed to get all tablet id list", K(ret)); + } else if (OB_FAIL(prepare_mview_dep_tablet_set_(param, backup_data_type))) { + LOG_WARN("failed to prepare mview dep tablet set", K(ret), K(param), K(backup_data_type)); + } else if (OB_FAIL(check_mview_tablet_set_(param.tenant_id_, sql_proxy))) { + LOG_WARN("failed to check mview tablet set", K(ret), K(param)); } else if (tablet_list.empty()) { ret = OB_NO_TABLET_NEED_BACKUP; LOG_WARN("no tablet in log stream need to backup", K(ret), K(param)); @@ -1509,6 +1518,34 @@ int ObLSBackupCtx::finish_task(const int64_t file_id) return ret; } +int ObLSBackupCtx::check_is_major_compaction_mview_dep_tablet(const common::ObTabletID &tablet_id, bool &is_dep) const +{ + int ret = OB_SUCCESS; + is_dep = false; + if (!tablet_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid args", K(ret), K(tablet_id)); + } else { + int hash_ret = mview_dep_tablet_set_.exist_refactored(tablet_id); + if (OB_HASH_EXIST == hash_ret) { + is_dep = true; + LOG_INFO("is mview dependent tablet", K(tablet_id)); + } else if (OB_HASH_NOT_EXIST == hash_ret) { + is_dep = false; + } else { + ret = hash_ret; + LOG_WARN("failed to check is mview dep tablet", K(ret), K(tablet_id)); + } + } + return ret; +} + +void ObLSBackupCtx::add_wait_reuse_across_sstable_time(int64_t cost_time) +{ + ObMutexGuard guard(mutex_); + wait_reuse_across_sstable_time_ += cost_time; +} + int ObLSBackupCtx::recover_last_retry_ctx_() { int ret = OB_SUCCESS; @@ -1566,6 +1603,71 @@ int ObLSBackupCtx::get_all_tablet_id_list_( return ret; } +int ObLSBackupCtx::get_backup_scn_(const ObLSBackupParam ¶m, + const share::ObBackupDataType &backup_data_type, share::SCN &backup_scn) +{ + int ret = OB_SUCCESS; + ObBackupDataStore store; + ObBackupDataLSAttrDesc ls_info; + ObBackupSetTaskAttr task_attr; + if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy should not be null", K(ret)); + } else if (OB_FAIL(ObBackupTaskOperator::get_backup_task( + *sql_proxy_, param.job_id_, param.tenant_id_, false, task_attr))) { + LOG_WARN("failed to get backup task", K(ret), K_(param)); + } else if (OB_FAIL(store.init(param.backup_dest_, param.backup_set_desc_))) { + LOG_WARN("failed to init backup data source", K(ret), K(param)); + } else if (OB_FAIL(store.read_ls_attr_info(task_attr.meta_turn_id_, ls_info))) { + LOG_WARN("failed to read ls attr info", K(ret)); + } else { + backup_scn = ls_info.backup_scn_; + LOG_INFO("get backup scn", K(param), K(backup_data_type), K(backup_scn)); + } + return ret; +} + +int ObLSBackupCtx::prepare_mview_dep_tablet_set_(const ObLSBackupParam ¶m, + const share::ObBackupDataType &backup_data_type) +{ + int ret = OB_SUCCESS; + ObBackupDataStore store; + ObBackupMajorCompactionMViewDepTabletListDesc desc; + if (backup_data_type.is_sys_backup()) { + // do nothing + } else if (OB_FAIL(store.init(param.backup_dest_, param.backup_set_desc_))) { + LOG_WARN("failed to init backup data source", K(ret), K(param)); + } else if (OB_FAIL(store.read_major_compaction_mview_dep_tablet_list(desc))) { + LOG_WARN("failed to read mview dep tablet list", K(ret)); + } else { + const common::ObSArray &mview_tablet_list = desc.tablet_id_list_;; + ARRAY_FOREACH_X(mview_tablet_list, idx, cnt, OB_SUCC(ret)) { + const ObTabletID &tablet_id = mview_tablet_list.at(idx); + const int flag = 1; + if (OB_FAIL(mview_dep_tablet_set_.set_refactored(tablet_id, flag))) { + LOG_WARN("failed to set refactored", K(ret), K(tablet_id)); + } + } + } + return ret; +} + +// This is only a best effort defense, which do not consider primary-standby switching during backup +int ObLSBackupCtx::check_mview_tablet_set_(const uint64_t tenant_id, common::ObMySQLProxy &sql_proxy) +{ + int ret = OB_SUCCESS; + ObAllTenantInfo tenant_info; + if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id, &sql_proxy, false/*for update*/, tenant_info))) { + LOG_WARN("failed to get tenant info", K(ret), K(tenant_id)); + } else if (tenant_info.is_primary()) { + LOG_INFO("tenant is primary", K(tenant_id), K(tenant_info)); + } else if (!mview_dep_tablet_set_.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant not primary but mview dep tablet is not empty", K(tenant_id), K(tenant_info)); + } + return ret; +} + int ObLSBackupCtx::seperate_tablet_id_list_(const common::ObIArray &tablet_id_list, common::ObArray &sys_tablet_id_list, common::ObArray &data_tablet_id_list) { diff --git a/src/storage/backup/ob_backup_ctx.h b/src/storage/backup/ob_backup_ctx.h index 56857c4872..8bc0f2a994 100644 --- a/src/storage/backup/ob_backup_ctx.h +++ b/src/storage/backup/ob_backup_ctx.h @@ -30,6 +30,7 @@ #include "storage/blocksstable/ob_logic_macro_id.h" #include "storage/backup/ob_backup_file_writer_ctx.h" #include "lib/oblog/ob_log_module.h" +#include "lib/hash/ob_hashset.h" namespace oceanbase { namespace backup { @@ -299,10 +300,16 @@ public: return tablet_holder_; } + int check_is_major_compaction_mview_dep_tablet(const common::ObTabletID &tablet_id, bool &is_major_compaction_mview_dep_tablet) const; + void add_wait_reuse_across_sstable_time(int64_t cost_time); + private: int recover_last_retry_ctx_(); int prepare_tablet_id_reader_(ObILSTabletIdReader *&reader); int get_all_tablet_id_list_(ObILSTabletIdReader *reader, common::ObIArray &tablet_list); + int get_backup_scn_(const ObLSBackupParam ¶m, const share::ObBackupDataType &backup_data_type, share::SCN &backup_scn); + int prepare_mview_dep_tablet_set_(const ObLSBackupParam ¶m, const share::ObBackupDataType &backup_data_type); + int check_mview_tablet_set_(const uint64_t tenant_id, common::ObMySQLProxy &sql_proxy); int seperate_tablet_id_list_(const common::ObIArray &tablet_id_list, common::ObArray &sys_tablet_list, common::ObArray &data_tablet_id_list); int inner_do_next_(common::ObTabletID &tablet_id); @@ -312,6 +319,7 @@ private: private: static const int64_t DEFAULT_FINISH_TABLET_SET_SIZE = 10000; + static const int64_t DEFAULT_MVIEW_DEP_TABLET_SET_SIZE = 10000; public: bool is_inited_; @@ -338,6 +346,8 @@ public: share::SCN backup_tx_table_filled_tx_scn_; ObBackupTabletIndexBlockBuilderMgr index_builder_mgr_; common::ObInOutBandwidthThrottle *bandwidth_throttle_; + common::hash::ObHashSet mview_dep_tablet_set_; + int64_t wait_reuse_across_sstable_time_; DISALLOW_COPY_AND_ASSIGN(ObLSBackupCtx); }; diff --git a/src/storage/backup/ob_backup_data_store.cpp b/src/storage/backup/ob_backup_data_store.cpp index 965288eb0c..986d729e5c 100644 --- a/src/storage/backup/ob_backup_data_store.cpp +++ b/src/storage/backup/ob_backup_data_store.cpp @@ -223,6 +223,30 @@ bool ObBackupTableListMetaInfoDesc::is_valid() const return scn_.is_valid() && count_ >= 0 && batch_size_ > 0; } +/* + *-----------------------------ObBackupMajorCompactionMViewDepTabletListDesc----------------------- + */ + +OB_SERIALIZE_MEMBER(ObBackupMajorCompactionMViewDepTabletListDesc, tablet_id_list_); + +ObBackupMajorCompactionMViewDepTabletListDesc::ObBackupMajorCompactionMViewDepTabletListDesc() + : ObExternBackupDataDesc(ObBackupFileType::BACKUP_MVIEW_DEP_TABLET_LIST_FILE, FILE_VERSION), + tablet_id_list_() {} + +bool ObBackupMajorCompactionMViewDepTabletListDesc::is_valid() const +{ + int ret = OB_SUCCESS; + bool bret = true; + ARRAY_FOREACH(tablet_id_list_, i) { + const ObTabletID &tablet_id = tablet_id_list_.at(i); + if (!tablet_id.is_valid()) { + bret = false; + break; + } + } + return bret; +} + int ObBackupSetFilter::get_backup_set_array(ObIArray &backup_set_array) const { int ret = OB_SUCCESS; @@ -1461,3 +1485,46 @@ int ObBackupDataStore::is_table_list_meta_exist(const share::SCN &scn, bool &is_ } return ret; } + +int ObBackupDataStore::write_major_compaction_mview_dep_tablet_list(const ObBackupMajorCompactionMViewDepTabletListDesc &desc) +{ + int ret = OB_SUCCESS; + share::ObBackupPath path; + ObBackupPathString full_path; + if (!is_init()) { + ret = OB_NOT_INIT; + LOG_WARN("backup data extern mgr not init", K(ret)); + } else if (!desc.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("table list meta desc is not valid", K(ret), K(desc)); + } else if (OB_FAIL(ObBackupPathUtil::get_major_compaction_mview_dep_tablet_list_path(backup_set_dest_, path))) { + LOG_WARN("fail to get table list meta path", K(ret), K(backup_set_dest_)); + } else if (OB_FAIL(full_path.assign(path.get_obstr()))) { + LOG_WARN("fail to assign full path", K(ret)); + } else if (OB_FAIL(write_single_file(full_path, desc))) { + LOG_WARN("fail to write single file", K(ret), K(desc)); + } else { + LOG_INFO("write mview dep tablet list", K(desc)); + } + return ret; +} + +int ObBackupDataStore::read_major_compaction_mview_dep_tablet_list(ObBackupMajorCompactionMViewDepTabletListDesc &desc) +{ + int ret = OB_SUCCESS; + ObBackupPath path; + ObBackupPathString full_path; + if (!is_init()) { + ret = OB_NOT_INIT; + LOG_WARN("backup data extern mgr not init", K(ret)); + } else if (OB_FAIL(ObBackupPathUtil::get_major_compaction_mview_dep_tablet_list_path(backup_set_dest_, path))) { + LOG_WARN("fail to get table list dir path", K(ret), K_(backup_set_dest)); + } else if (OB_FAIL(full_path.assign(path.get_obstr()))) { + LOG_WARN("fail to assign full path", K(ret)); + } else if (OB_FAIL(read_single_file(full_path, desc))) { + LOG_WARN("fail to read single file", K(ret), K(full_path)); + } else { + LOG_INFO("read mview dep tablet list", K(desc)); + } + return ret; +} diff --git a/src/storage/backup/ob_backup_data_store.h b/src/storage/backup/ob_backup_data_store.h index 4352d6db68..7e9bf1824b 100644 --- a/src/storage/backup/ob_backup_data_store.h +++ b/src/storage/backup/ob_backup_data_store.h @@ -295,6 +295,21 @@ private: DISALLOW_COPY_AND_ASSIGN(ObBackupTableListMetaInfoDesc); }; +class ObBackupMajorCompactionMViewDepTabletListDesc final : public ObExternBackupDataDesc +{ +public: + static const uint8_t FILE_VERSION = 1; + OB_UNIS_VERSION(1); +public: + ObBackupMajorCompactionMViewDepTabletListDesc(); + ~ObBackupMajorCompactionMViewDepTabletListDesc() {} + bool is_valid() const override; + TO_STRING_KV(K_(tablet_id_list)); +public: + ObSArray tablet_id_list_; +private: + DISALLOW_COPY_AND_ASSIGN(ObBackupMajorCompactionMViewDepTabletListDesc); +}; class ObBackupSetFilter : public ObBaseDirEntryOperator { @@ -383,6 +398,9 @@ public: int write_table_list_meta_info(const share::SCN &scn, const ObBackupTableListMetaInfoDesc &desc); int read_table_list_file(const char* file_name, ObBackupPartialTableListDesc &desc); int is_table_list_meta_exist(const share::SCN &scn, bool &is_exist); + // mview dep tablet list + int write_major_compaction_mview_dep_tablet_list(const ObBackupMajorCompactionMViewDepTabletListDesc &desc); + int read_major_compaction_mview_dep_tablet_list(ObBackupMajorCompactionMViewDepTabletListDesc &desc); TO_STRING_KV(K_(backup_desc)); public: diff --git a/src/storage/backup/ob_backup_data_struct.cpp b/src/storage/backup/ob_backup_data_struct.cpp index 00d4c70f67..f525264096 100644 --- a/src/storage/backup/ob_backup_data_struct.cpp +++ b/src/storage/backup/ob_backup_data_struct.cpp @@ -771,7 +771,6 @@ void ObBackupTabletMeta::reset() } /* ObBackupSSTableMeta */ - OB_SERIALIZE_MEMBER(ObBackupSSTableMeta, tablet_id_, sstable_meta_, logic_id_list_, entry_block_addr_for_other_block_, // FARM COMPAT WHITELIST total_other_block_count_, // FARM COMPAT WHITELIST diff --git a/src/storage/backup/ob_backup_data_struct.h b/src/storage/backup/ob_backup_data_struct.h index 0c53625d0a..bf8d9d6fc8 100644 --- a/src/storage/backup/ob_backup_data_struct.h +++ b/src/storage/backup/ob_backup_data_struct.h @@ -574,13 +574,14 @@ public: int assign(const ObBackupSSTableMeta &backup_sstable_meta); TO_STRING_KV(K_(tablet_id), K_(sstable_meta), K_(logic_id_list), - K_(entry_block_addr_for_other_block), K_(total_other_block_count), K_(is_major_compaction_mview_dep)); + K_(entry_block_addr_for_other_block), K_(total_other_block_count), + K_(is_major_compaction_mview_dep)); common::ObTabletID tablet_id_; blocksstable::ObMigrationSSTableParam sstable_meta_; common::ObSArray logic_id_list_; ObBackupLinkedBlockAddr entry_block_addr_for_other_block_; int64_t total_other_block_count_; - bool is_major_compaction_mview_dep_; // placeholder for feature collect_mv + bool is_major_compaction_mview_dep_; }; struct ObBackupMacroBlockIDPair final { diff --git a/src/storage/backup/ob_backup_index_block_builder_mgr.cpp b/src/storage/backup/ob_backup_index_block_builder_mgr.cpp index ac032df2d9..ccbf8c5e6a 100644 --- a/src/storage/backup/ob_backup_index_block_builder_mgr.cpp +++ b/src/storage/backup/ob_backup_index_block_builder_mgr.cpp @@ -602,6 +602,8 @@ int ObBackupTabletSSTableIndexBuilderMgr::init(const uint64_t tenant_id, const c LOG_WARN("failed to reserve merge res", K(ret)); } else if (OB_FAIL(sstable_ready_list_.prepare_allocate(table_key_array.count()))) { LOG_WARN("failed to reserve merge res", K(ret)); + } else if (OB_FAIL(local_reuse_map_.create(BUCKET_NUM, ObModIds::BACKUP))) { + LOG_WARN("failed to create local reuse map", K(ret)); } else { ARRAY_FOREACH(sstable_ready_list_, idx) { sstable_ready_list_.at(idx) = false; @@ -945,5 +947,85 @@ int ObBackupTabletSSTableIndexBuilderMgr::close_sstable_index_builder_( return ret; } +int ObBackupTabletSSTableIndexBuilderMgr::insert_place_holder_macro_index( + const blocksstable::ObLogicMacroBlockId &logic_id) +{ + int ret = OB_SUCCESS; + ObMutexGuard guard(mutex_); + ObBackupMacroBlockIndex tmp_index; + int32_t hash_ret = local_reuse_map_.get_refactored(logic_id, tmp_index); + if (OB_HASH_NOT_EXIST != hash_ret) { + LOG_WARN("macro index already exist, do nothing", K(ret), K(logic_id)); + } else { + ObBackupMacroBlockIndex macro_index; + macro_index.reset(); + if (OB_FAIL(local_reuse_map_.set_refactored(logic_id, macro_index))) { + LOG_WARN("failed to set macro index", K(ret), K(logic_id), K(macro_index)); + } else { + LOG_INFO("insert place holder macro index", K(logic_id)); + } + } + return ret; +} + +int ObBackupTabletSSTableIndexBuilderMgr::update_logic_id_to_macro_index( + const blocksstable::ObLogicMacroBlockId &logic_id, const ObBackupMacroBlockIndex ¯o_index) +{ + int ret = OB_SUCCESS; + ObMutexGuard guard(mutex_); + ObBackupMacroBlockIndex tmp_index; + int32_t hash_ret = local_reuse_map_.get_refactored(logic_id, tmp_index); + if (!logic_id.is_valid() || !macro_index.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid args", K(ret), K(logic_id), K(macro_index)); + } else { + if (OB_HASH_NOT_EXIST == hash_ret) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("previous logic id do not exist", K(ret), K(hash_ret), K(logic_id)); + } else if (OB_FAIL(local_reuse_map_.set_refactored(logic_id, macro_index, 1))) { + LOG_WARN("failed to set macro index", K(ret)); + } else { + LOG_INFO("update logic id to macro index", K(logic_id), K(macro_index)); + } + } + return ret; +} + +int ObBackupTabletSSTableIndexBuilderMgr::check_place_holder_macro_index_exist( + const blocksstable::ObLogicMacroBlockId &logic_id, bool &exist) +{ + int ret = OB_SUCCESS; + exist = false; + ObMutexGuard guard(mutex_); + ObBackupMacroBlockIndex tmp_index; + int32_t hash_ret = local_reuse_map_.get_refactored(logic_id, tmp_index); + if (OB_HASH_NOT_EXIST == hash_ret) { + exist = false; + } else if (!tmp_index.is_valid()) { + exist = true; + } + return ret; +} + +int ObBackupTabletSSTableIndexBuilderMgr::check_real_macro_index_exist( + const blocksstable::ObLogicMacroBlockId &logic_id, bool &exist, ObBackupMacroBlockIndex &index) +{ + int ret = OB_SUCCESS; + exist = false; + ObMutexGuard guard(mutex_); + ObBackupMacroBlockIndex tmp_index; + int32_t hash_ret = local_reuse_map_.get_refactored(logic_id, tmp_index); + if (OB_HASH_NOT_EXIST == hash_ret) { + exist = false; + } else if (!tmp_index.is_valid()) { + exist = false; + } else { + exist = true; + index = tmp_index; + } + LOG_INFO("check macro index exist", K(logic_id), K(exist), K(index)); + return ret; +} + } } diff --git a/src/storage/backup/ob_backup_index_block_builder_mgr.h b/src/storage/backup/ob_backup_index_block_builder_mgr.h index b25917ecab..7888ef5496 100644 --- a/src/storage/backup/ob_backup_index_block_builder_mgr.h +++ b/src/storage/backup/ob_backup_index_block_builder_mgr.h @@ -136,6 +136,15 @@ private: int close_sstable_index_builder_(blocksstable::ObSSTableIndexBuilder *index_builder, ObIODevice *device_handle, blocksstable::ObSSTableMergeRes &merge_res); +public: + int insert_place_holder_macro_index(const blocksstable::ObLogicMacroBlockId &logic_id); + int update_logic_id_to_macro_index(const blocksstable::ObLogicMacroBlockId &logic_id, const ObBackupMacroBlockIndex &index); + int check_place_holder_macro_index_exist(const blocksstable::ObLogicMacroBlockId &logic_id, bool &exist); + int check_real_macro_index_exist(const blocksstable::ObLogicMacroBlockId &logic_id, bool &exist, ObBackupMacroBlockIndex &index); + +private: + static const int64_t BUCKET_NUM = 10000; + private: bool is_inited_; lib::ObMutex mutex_; @@ -144,6 +153,7 @@ private: common::ObArray builders_; common::ObArray merge_results_; common::ObArray sstable_ready_list_; + common::hash::ObHashMap local_reuse_map_; DISALLOW_COPY_AND_ASSIGN(ObBackupTabletSSTableIndexBuilderMgr); }; diff --git a/src/storage/backup/ob_backup_reader.cpp b/src/storage/backup/ob_backup_reader.cpp index bbff29c24a..57ca07c41b 100644 --- a/src/storage/backup/ob_backup_reader.cpp +++ b/src/storage/backup/ob_backup_reader.cpp @@ -645,7 +645,7 @@ int ObTabletMetaBackupReader::get_meta_data(blocksstable::ObBufferReader &buffer ObSSTableMetaBackupReader::ObSSTableMetaBackupReader() : ObITabletMetaBackupReader(), cond_(), sstable_array_(), buffer_writer_("BackupReader"), table_store_wrapper_(), - linked_writer_(NULL) + linked_writer_(NULL), is_major_compaction_mview_dep_(false) {} ObSSTableMetaBackupReader::~ObSSTableMetaBackupReader() @@ -670,15 +670,19 @@ int ObSSTableMetaBackupReader::init(const common::ObTabletID &tablet_id, tablet_handle_ = &tablet_handle; linked_writer_ = linked_writer; ObTablet &tablet = *tablet_handle_->get_obj(); + bool is_major_compaction_mview_dep = false; if (OB_FAIL(tablet.fetch_table_store(table_store_wrapper_))) { LOG_WARN("failed to fetch table store from tablet", K(ret)); + } else if (OB_FAIL(ls_backup_ctx.check_is_major_compaction_mview_dep_tablet(tablet_id, is_major_compaction_mview_dep))) { + LOG_WARN("failed to check is mview dep tablet", K(ret), K(tablet_id)); } else if (OB_FAIL(ObBackupUtils::get_sstables_by_data_type( - tablet_handle, backup_data_type, *table_store_wrapper_.get_member(), sstable_array_))) { + tablet_handle, backup_data_type, *table_store_wrapper_.get_member(), is_major_compaction_mview_dep, sstable_array_))) { LOG_WARN("failed to get sstables by data type", K(ret), K(tablet_handle)); } else { builder_mgr_ = &index_block_builder_mgr; ls_backup_ctx_ = &ls_backup_ctx; device_handle_ = device_handle; + is_major_compaction_mview_dep_ = is_major_compaction_mview_dep; is_inited_ = true; } } @@ -708,6 +712,7 @@ int ObSSTableMetaBackupReader::get_meta_data(blocksstable::ObBufferReader &buffe ObTablet *tablet = tablet_handle_->get_obj(); ObBackupSSTableMeta backup_sstable_meta; backup_sstable_meta.tablet_id_ = tablet_id_; + backup_sstable_meta.is_major_compaction_mview_dep_ = is_major_compaction_mview_dep_; blocksstable::ObSSTableMergeRes *merge_res = NULL; if (GCTX.is_shared_storage_mode() && table_key.is_ddl_dump_sstable()) { if (FAILEDx(get_macro_block_id_list_(*sstable_ptr, backup_sstable_meta))) { diff --git a/src/storage/backup/ob_backup_reader.h b/src/storage/backup/ob_backup_reader.h index abe110c0ef..8619a73993 100644 --- a/src/storage/backup/ob_backup_reader.h +++ b/src/storage/backup/ob_backup_reader.h @@ -300,6 +300,7 @@ private: blocksstable::ObSelfBufferWriter buffer_writer_; ObTabletMemberWrapper table_store_wrapper_; ObBackupLinkedBlockItemWriter *linked_writer_; + bool is_major_compaction_mview_dep_; DISALLOW_COPY_AND_ASSIGN(ObSSTableMetaBackupReader); }; diff --git a/src/storage/backup/ob_backup_task.cpp b/src/storage/backup/ob_backup_task.cpp index bfe58132a6..e750ee8038 100644 --- a/src/storage/backup/ob_backup_task.cpp +++ b/src/storage/backup/ob_backup_task.cpp @@ -2891,13 +2891,15 @@ int ObLSBackupDataTask::do_backup_macro_block_data_(common::ObIArraytablet_stat_)) { } else if (OB_FAIL(tablet_stat->add_opened_rebuilder_count(tablet_id))) { LOG_WARN("failed to add opened rebuilder count", K(ret), K(tablet_id)); + } else { + LOG_INFO("add opened rebuilder count", K(tablet_id), "table_key", item.get_table_key(), K(item)); } return ret; } @@ -4077,6 +4102,8 @@ int ObLSBackupDataTask::close_index_block_rebuilder_if_need_(const ObBackupProvi } else if (FALSE_IT(tablet_stat = &ls_backup_ctx_->tablet_stat_)) { } else if (OB_FAIL(tablet_stat->add_closed_rebuilder_count(tablet_id))) { LOG_WARN("failed to add closed rebuilder count", K(ret), K(tablet_id)); + } else { + LOG_INFO("close rebuilder count", K(tablet_id), "table_key", item.get_table_key(), K(item)); } return ret; } @@ -4137,6 +4164,87 @@ int ObLSBackupDataTask::close_tree_device_handle_( return ret; } +int ObLSBackupDataTask::update_logic_id_to_macro_index_(const common::ObTabletID &tablet_id, const storage::ObITable::TableKey &table_key, + const blocksstable::ObLogicMacroBlockId &logic_id, const ObBackupMacroBlockIndex ¯o_index) +{ + int ret = OB_SUCCESS; + ObBackupTabletSSTableIndexBuilderMgr *mgr = NULL; + if (OB_ISNULL(index_builder_mgr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mgr should not be null", K(ret)); + } else if (!logic_id.is_valid() || !macro_index.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid args", K(ret), K(logic_id), K(macro_index)); + } else if (!table_key.is_major_sstable()) { + // do nothing + } else if (OB_FAIL(index_builder_mgr_->get_sstable_index_builder_mgr(tablet_id, mgr))) { + LOG_WARN("failed to get sstable index builder mgr", K(ret), K(tablet_id)); + } else if (OB_FAIL(mgr->update_logic_id_to_macro_index(logic_id, macro_index))) { + LOG_WARN("failed to update logic id to macro index", K(ret), K(logic_id)); + } + return ret; +} + +int ObLSBackupDataTask::wait_reuse_other_block_ready_( + const common::ObTabletID &tablet_id, const blocksstable::ObLogicMacroBlockId &logic_id, + ObBackupMacroBlockIndex ¯o_index) +{ + int ret = OB_SUCCESS; + macro_index.reset(); + bool is_ready = false; + static const int64_t DEFAULT_SLEEP_US = 10_ms; + int64_t start_time = ObTimeUtility::current_time(); + while (OB_SUCC(ret)) { + is_ready = false; + if (OB_ISNULL(ls_backup_ctx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls backup ctx should not be null", K(ret)); + } else if (OB_SUCCESS != ls_backup_ctx_->get_result_code()) { + ret = OB_EAGAIN; + LOG_WARN("ctx already failed", K(ret)); + } else if (OB_FAIL(inner_check_reuse_block_ready_(tablet_id, logic_id, macro_index, is_ready))) { + LOG_WARN("failed to inner check reuse block ready", K(ret), K(tablet_id), K(logic_id)); + } else if (is_ready) { + LOG_INFO("reuse macro block is ready", K(tablet_id), K(logic_id), K(macro_index)); +#ifdef ERRSIM + SERVER_EVENT_SYNC_ADD("backup_data", "reuse_macro_block", + "tablet_id", tablet_id.id(), + "logic_id", logic_id, + "macro_index", macro_index); +#endif + break; + } else { + usleep(DEFAULT_SLEEP_US); + } + } + if (OB_SUCC(ret) && OB_NOT_NULL(ls_backup_ctx_)) { + ls_backup_ctx_->add_wait_reuse_across_sstable_time(ObTimeUtility::current_time() - start_time); + } + return ret; +} + +int ObLSBackupDataTask::inner_check_reuse_block_ready_( + const common::ObTabletID &tablet_id, const blocksstable::ObLogicMacroBlockId &logic_id, + ObBackupMacroBlockIndex ¯o_index, bool &is_ready) +{ + int ret = OB_SUCCESS; + macro_index.reset(); + is_ready = false; + ObBackupTabletSSTableIndexBuilderMgr *mgr = NULL; + if (OB_ISNULL(index_builder_mgr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mgr should not be null", K(ret)); + } else if (!tablet_id.is_valid() || !logic_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid args", K(ret), K(tablet_id), K(logic_id)); + } else if (OB_FAIL(index_builder_mgr_->get_sstable_index_builder_mgr(tablet_id, mgr))) { + LOG_WARN("failed to get sstable index builder mgr", K(ret), K(tablet_id)); + } else if (OB_FAIL(mgr->check_real_macro_index_exist(logic_id, is_ready, macro_index))) { + LOG_WARN("failed to check macro index exist", K(ret), K(logic_id)); + } + return ret; +} + /* ObLSBackupMetaTask */ ObLSBackupMetaTask::ObLSBackupMetaTask() @@ -5342,6 +5450,11 @@ int ObLSBackupFinishTask::process() } else if (FALSE_IT(result = ls_backup_ctx_->get_result_code())) { } else { ls_backup_ctx_->stat_mgr_.print_stat(); + SERVER_EVENT_ADD("backup_data", "report_wait_reuse_across_sstable_time", + "tenant_id", tenant_id, + "task_id", task_id, + "ls_id", ls_id.id(), + "wait_reuse_time", ls_backup_ctx_->wait_reuse_across_sstable_time_); } #ifdef ERRSIM if (ls_backup_ctx_->param_.ls_id_.id() == GCONF.errsim_backup_ls_id) { diff --git a/src/storage/backup/ob_backup_task.h b/src/storage/backup/ob_backup_task.h index fa3ca56829..1c9a3fca69 100644 --- a/src/storage/backup/ob_backup_task.h +++ b/src/storage/backup/ob_backup_task.h @@ -621,6 +621,12 @@ private: int remove_index_builders_(); int remove_sstable_index_builder_(const common::ObTabletID &tablet_id); int close_tree_device_handle_(ObBackupWrapperIODevice *&index_tree_device_handle, ObBackupWrapperIODevice *&meta_tree_device_handle); + int update_logic_id_to_macro_index_(const common::ObTabletID &tablet_id, const storage::ObITable::TableKey &table_key, + const blocksstable::ObLogicMacroBlockId &logic_id, const ObBackupMacroBlockIndex ¯o_index); + int wait_reuse_other_block_ready_(const common::ObTabletID &tablet_id, + const blocksstable::ObLogicMacroBlockId &logic_id, ObBackupMacroBlockIndex ¯o_index); + int inner_check_reuse_block_ready_(const common::ObTabletID &tablet_id, + const blocksstable::ObLogicMacroBlockId &logic_id, ObBackupMacroBlockIndex ¯o_index, bool &is_ready); private: static const int64_t CHECK_DISK_SPACE_INTERVAL = 5 * 1000 * 1000; // 5s; diff --git a/src/storage/backup/ob_backup_utils.cpp b/src/storage/backup/ob_backup_utils.cpp index 01b9632f87..d295dbb1cc 100644 --- a/src/storage/backup/ob_backup_utils.cpp +++ b/src/storage/backup/ob_backup_utils.cpp @@ -83,7 +83,7 @@ int ObBackupUtils::calc_start_replay_scn( } int ObBackupUtils::get_sstables_by_data_type(const storage::ObTabletHandle &tablet_handle, const share::ObBackupDataType &backup_data_type, - const storage::ObTabletTableStore &tablet_table_store, common::ObIArray &sstable_array) + const storage::ObTabletTableStore &tablet_table_store, const bool is_major_compaction_mview_dep_tablet, common::ObIArray &sstable_array) { int ret = OB_SUCCESS; sstable_array.reset(); @@ -97,7 +97,7 @@ int ObBackupUtils::get_sstables_by_data_type(const storage::ObTabletHandle &tabl } else { if (OB_FAIL(fetch_minor_and_ddl_sstables_(tablet_handle, tablet_table_store, sstable_array))) { LOG_WARN("failed to fetch minor and ddl sstables", K(ret), K(tablet_handle), K(tablet_table_store)); - } else if (OB_FAIL(fetch_major_sstables_(tablet_handle, tablet_table_store, sstable_array))) { + } else if (OB_FAIL(fetch_major_sstables_(tablet_handle, tablet_table_store, is_major_compaction_mview_dep_tablet, sstable_array))) { LOG_WARN("failed to fetch major sstables", K(ret), K(tablet_handle), K(tablet_table_store)); } } @@ -162,7 +162,8 @@ int ObBackupUtils::fetch_minor_and_ddl_sstables_(const storage::ObTabletHandle & } int ObBackupUtils::fetch_major_sstables_(const storage::ObTabletHandle &tablet_handle, - const storage::ObTabletTableStore &tablet_table_store, common::ObIArray &sstable_array) + const storage::ObTabletTableStore &tablet_table_store, const bool is_major_compaction_mview_dep_tablet, + common::ObIArray &sstable_array) { int ret = OB_SUCCESS; const storage::ObSSTableArray *major_sstable_array_ptr = NULL; @@ -187,16 +188,23 @@ int ObBackupUtils::fetch_major_sstables_(const storage::ObTabletHandle &tablet_h ret = OB_ERR_UNEXPECTED; LOG_WARN("medium list is invalid for last major sstable", K(ret), KPC(medium_info_list), KPC(last_major_sstable_ptr), K(tablet_handle)); - } else if (last_major_sstable_ptr->is_co_sstable()) { - if (OB_FAIL(static_cast(last_major_sstable_ptr)->get_all_tables(sstable_array))) { - LOG_WARN("failed to get all cg tables from co table", K(ret), KPC(last_major_sstable_ptr)); + } else if (is_major_compaction_mview_dep_tablet) { + if (OB_FAIL(major_sstable_array_ptr->get_all_table_wrappers(sstable_array))) { + LOG_WARN("failed to get all table wrappers", K(ret)); + } + } else { // not mview dep tablet + if (last_major_sstable_ptr->is_co_sstable()) { + if (OB_FAIL(static_cast(last_major_sstable_ptr)->get_all_tables(sstable_array))) { + LOG_WARN("failed to get all cg tables from co table", K(ret), KPC(last_major_sstable_ptr)); + } + } else { + if (OB_FAIL(major_wrapper.set_sstable(static_cast(last_major_sstable_ptr)))) { + LOG_WARN("failed to set major wrapper", K(ret), KPC(last_major_sstable_ptr)); + } else if (OB_FAIL(sstable_array.push_back(major_wrapper))) { + LOG_WARN("failed to push back", K(ret), K(major_wrapper)); + } } - } else if (OB_FAIL(major_wrapper.set_sstable(static_cast(last_major_sstable_ptr)))) { - LOG_WARN("failed to set major wrapper", K(ret), KPC(last_major_sstable_ptr)); - } else if (OB_FAIL(sstable_array.push_back(major_wrapper))) { - LOG_WARN("failed to push back", K(ret), K(major_wrapper)); } - return ret; } @@ -1290,7 +1298,7 @@ ObBackupProviderItem::ObBackupProviderItem() table_key_(), tablet_id_(), nested_offset_(0), nested_size_(0), timestamp_(0), need_copy_(true), macro_index_(), - absolute_row_offset_(0) + absolute_row_offset_(0), need_reuse_across_sstable_(false) {} ObBackupProviderItem::~ObBackupProviderItem() @@ -1373,7 +1381,8 @@ bool ObBackupProviderItem::operator==(const ObBackupProviderItem &other) const return item_type_ == other.item_type_ && backup_data_type_ == other.backup_data_type_ && logic_id_ == other.logic_id_ && macro_block_id_ == other.macro_block_id_ && table_key_ == other.table_key_ && tablet_id_ == other.tablet_id_ && nested_size_ == other.nested_size_ && nested_offset_ == other.nested_offset_ - && need_copy_ == other.need_copy_ && macro_index_ == other.macro_index_ && absolute_row_offset_ == other.absolute_row_offset_; + && need_copy_ == other.need_copy_ && macro_index_ == other.macro_index_ && absolute_row_offset_ == other.absolute_row_offset_ + && need_reuse_across_sstable_ == other.need_reuse_across_sstable_; } bool ObBackupProviderItem::operator!=(const ObBackupProviderItem &other) const @@ -1440,6 +1449,7 @@ int ObBackupProviderItem::deep_copy(const ObBackupProviderItem &src) need_copy_ = src.need_copy_; macro_index_ = src.macro_index_; absolute_row_offset_ = src.absolute_row_offset_; + need_reuse_across_sstable_ = src.need_reuse_across_sstable_; return ret; } @@ -1458,6 +1468,7 @@ int ObBackupProviderItem::deep_copy(const ObBackupProviderItem &src, char *buf, need_copy_ = src.need_copy_; macro_index_ = src.macro_index_; absolute_row_offset_ = src.absolute_row_offset_; + need_reuse_across_sstable_ = src.need_reuse_across_sstable_; return ret; } @@ -1488,6 +1499,7 @@ void ObBackupProviderItem::reset() need_copy_ = true; macro_index_.reset(); absolute_row_offset_ = 0; + need_reuse_across_sstable_ = false; } ObITable::TableKey ObBackupProviderItem::get_fake_table_key_() @@ -1791,6 +1803,7 @@ int ObBackupTabletProvider::prepare_tablet_(const uint64_t tenant_id, const shar bool can_explain = false; ObTabletMemberWrapper table_store_wrapper; bool need_skip_tablet = false; + bool is_major_compaction_mview_dep_tablet = false; bool is_split_dst = false; if (OB_ISNULL(ls_backup_ctx_)) { ret = OB_ERR_UNEXPECTED; @@ -1835,8 +1848,10 @@ int ObBackupTabletProvider::prepare_tablet_(const uint64_t tenant_id, const shar LOG_WARN("failed to check tablet replica validity", K(ret), K(tenant_id), K(ls_id), K(tablet_id), K(backup_data_type)); } else if (OB_FAIL(tablet_ref->tablet_handle_.get_obj()->fetch_table_store(table_store_wrapper))) { LOG_WARN("fail to fetch table store", K(ret)); + } else if (OB_FAIL(ls_backup_ctx_->check_is_major_compaction_mview_dep_tablet(tablet_id, is_major_compaction_mview_dep_tablet))) { + LOG_WARN("failed to check is mview dep tablet", K(ret)); } else if (OB_FAIL(fetch_tablet_sstable_array_( - tablet_id, tablet_ref->tablet_handle_, *table_store_wrapper.get_member(), backup_data_type, sstable_array))) { + tablet_id, tablet_ref->tablet_handle_, *table_store_wrapper.get_member(), backup_data_type, is_major_compaction_mview_dep_tablet, sstable_array))) { LOG_WARN("failed to fetch tablet sstable array", K(ret), K(tablet_id), KPC(tablet_ref), K(backup_data_type)); } else if (OB_FAIL(prepare_tablet_sstable_index_builders_(tablet_id, sstable_array))) { LOG_WARN("failed to prepare tablet sstable index builders", K(ret), K_(param), K(tablet_id), K(sstable_array)); @@ -2239,14 +2254,15 @@ int ObBackupTabletProvider::hold_tablet_handle_( int ObBackupTabletProvider::fetch_tablet_sstable_array_(const common::ObTabletID &tablet_id, const storage::ObTabletHandle &tablet_handle, const ObTabletTableStore &table_store, - const share::ObBackupDataType &backup_data_type, common::ObIArray &sstable_array) + const share::ObBackupDataType &backup_data_type, const bool is_major_compaction_mview_dep_tablet, + common::ObIArray &sstable_array) { int ret = OB_SUCCESS; sstable_array.reset(); if (!tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("get invalid args", K(ret), K(tablet_id)); - } else if (OB_FAIL(ObBackupUtils::get_sstables_by_data_type(tablet_handle, backup_data_type, table_store, sstable_array))) { + } else if (OB_FAIL(ObBackupUtils::get_sstables_by_data_type(tablet_handle, backup_data_type, table_store, is_major_compaction_mview_dep_tablet, sstable_array))) { LOG_WARN("failed to get sstables by data type", K(ret), K(tablet_handle), K(backup_data_type), K(table_store)); } else { LOG_INFO("fetch tablet sstable array", K(ret), K(tablet_id), K(backup_data_type), K(sstable_array)); @@ -2418,13 +2434,18 @@ int ObBackupTabletProvider::add_macro_block_id_item_list_(const common::ObTablet added_count = 0; for (int64_t i = 0; OB_SUCC(ret) && i < list.count(); ++i) { const ObBackupMacroBlockId ¯o_id = list.at(i); + const ObLogicMacroBlockId logic_id = macro_id.logic_id_; ObBackupProviderItem item; bool need_skip = false; share::ObBackupDataType backup_data_type; + bool need_reuse_across_sstable = false; if (OB_FAIL(get_backup_data_type_(table_key, backup_data_type))) { LOG_WARN("failed to get backup data type", K(ret), K(table_key)); + } else if (OB_FAIL(check_need_reuse_across_sstable_(tablet_id, table_key, logic_id, need_reuse_across_sstable))) { + LOG_WARN("failed to check need reuse local", K(ret), K(tablet_id), K(table_key), K(macro_id)); } else if (OB_FAIL(item.set(PROVIDER_ITEM_MACRO_ID, backup_data_type, macro_id, table_key, tablet_id))) { LOG_WARN("failed to set item", K(ret), K(macro_id), K(table_key), K(tablet_id)); + } else if (need_reuse_across_sstable && OB_FALSE_IT(item.set_need_reuse_across_sstable())) { } else if (!item.is_valid()) { ret = OB_INVALID_DATA; LOG_WARN("backup item is not valid", K(ret), K(item)); @@ -2720,6 +2741,31 @@ void ObBackupTabletProvider::free_queue_item_() } } +int ObBackupTabletProvider::check_need_reuse_across_sstable_(const common::ObTabletID &tablet_id, const storage::ObITable::TableKey &table_key, + const blocksstable::ObLogicMacroBlockId &logic_id, bool &need_reuse_across_sstable) +{ + int ret = OB_SUCCESS; + need_reuse_across_sstable = false; + ObBackupTabletIndexBlockBuilderMgr *mgr = index_builder_mgr_; + ObBackupTabletSSTableIndexBuilderMgr *sstable_mgr = NULL; + bool macro_index_exist = false; + if (!table_key.is_major_sstable()) { + need_reuse_across_sstable = false; + } else if (OB_FAIL(mgr->get_sstable_index_builder_mgr(tablet_id, sstable_mgr))) { + LOG_WARN("failed to get sstable index builder mgr", K(ret), K(tablet_id)); + } else if (OB_FAIL(sstable_mgr->check_place_holder_macro_index_exist(logic_id, macro_index_exist))) { + LOG_WARN("failed to check place holder macro index exist", K(ret), K(logic_id)); + } else if (macro_index_exist) { + need_reuse_across_sstable = true; + LOG_INFO("macro index exist, reuse local", K(tablet_id), K(table_key), K(logic_id)); + } else if (OB_FAIL(sstable_mgr->insert_place_holder_macro_index(logic_id))) { + LOG_WARN("failed to insert empty macro index", K(ret), K(logic_id)); + } else { + need_reuse_across_sstable = false; + } + return ret; +} + /* ObBackupMacroBlockTaskMgr */ ObBackupMacroBlockTaskMgr::ObBackupMacroBlockTaskMgr() diff --git a/src/storage/backup/ob_backup_utils.h b/src/storage/backup/ob_backup_utils.h index 013f450859..5179370bbf 100644 --- a/src/storage/backup/ob_backup_utils.h +++ b/src/storage/backup/ob_backup_utils.h @@ -55,7 +55,8 @@ class ObBackupMacroBlockIndexStore; class ObBackupUtils { public: static int get_sstables_by_data_type(const storage::ObTabletHandle &tablet_handle, const share::ObBackupDataType &backup_data_type, - const storage::ObTabletTableStore &table_store, common::ObIArray &sstable_array); + const storage::ObTabletTableStore &table_store, const bool is_major_compaction_mview_dep_tablet, + common::ObIArray &sstable_array); static int check_tablet_with_major_sstable(const storage::ObTabletHandle &tablet_handle, bool &with_major); static int fetch_macro_block_logic_id_list(const storage::ObTabletHandle &tablet_handle, const blocksstable::ObSSTable &sstable, common::ObIArray &logic_id_list); @@ -83,7 +84,8 @@ private: static int fetch_minor_and_ddl_sstables_(const storage::ObTabletHandle &tablet_handle, const storage::ObTabletTableStore &tablet_table_store, common::ObIArray &sstable_array); static int fetch_major_sstables_(const storage::ObTabletHandle &tablet_handle, - const storage::ObTabletTableStore &tablet_table_store, common::ObIArray &sstable_array); + const storage::ObTabletTableStore &tablet_table_store, const bool is_major_compaction_mview_dep_tablet, + common::ObIArray &sstable_array); }; struct ObBackupTabletCtx final { @@ -257,6 +259,8 @@ public: void set_no_need_copy() { need_copy_ = false; } bool get_need_copy() const { return need_copy_; } void set_macro_index(const ObBackupMacroBlockIndex ¯o_index) { macro_index_ = macro_index; } + void set_need_reuse_across_sstable() { need_reuse_across_sstable_ = true; } + bool get_need_reuse_across_sstable() const { return need_reuse_across_sstable_; } const ObBackupMacroBlockIndex &get_macro_index() { return macro_index_; } int64_t get_absolute_row_offset() const { return absolute_row_offset_; } TO_STRING_KV(K_(item_type), K_(backup_data_type), K_(logic_id), K_(macro_block_id), K_(table_key), K_(tablet_id), K_(nested_offset), K_(nested_size), K_(timestamp)); @@ -279,6 +283,7 @@ private: bool need_copy_; ObBackupMacroBlockIndex macro_index_; int64_t absolute_row_offset_; + bool need_reuse_across_sstable_; }; class ObBackupProviderItemCompare { @@ -352,7 +357,7 @@ private: int hold_tablet_handle_(const common::ObTabletID &tablet_id, ObBackupTabletHandleRef *tablet_handle); int fetch_tablet_sstable_array_(const common::ObTabletID &tablet_id, const storage::ObTabletHandle &tablet_handle, const ObTabletTableStore &table_store, const share::ObBackupDataType &backup_data_type, - common::ObIArray &sstable_array); + const bool is_major_compaction_mview_dep_tablet, common::ObIArray &sstable_array); int prepare_tablet_sstable_index_builders_(const common::ObTabletID &tablet_id, common::ObIArray &sstable_array); int open_tablet_sstable_index_builder_(const common::ObTabletID &tablet_id, const storage::ObTabletHandle &tablet_handle, @@ -382,6 +387,8 @@ private: int push_item_to_queue_(const ObBackupProviderItem &item); int pop_item_from_queue_(ObBackupProviderItem &item); void free_queue_item_(); + int check_need_reuse_across_sstable_(const common::ObTabletID &tablet_id, const storage::ObITable::TableKey &table_key, + const blocksstable::ObLogicMacroBlockId &logic_id, bool &need_reuse_across_sstable); private: static const int64_t BATCH_SIZE = 2000; diff --git a/src/storage/blocksstable/cs_encoding/ob_micro_block_cs_decoder.cpp b/src/storage/blocksstable/cs_encoding/ob_micro_block_cs_decoder.cpp index 0fe4351a59..e53559f3bf 100644 --- a/src/storage/blocksstable/cs_encoding/ob_micro_block_cs_decoder.cpp +++ b/src/storage/blocksstable/cs_encoding/ob_micro_block_cs_decoder.cpp @@ -1340,6 +1340,7 @@ int ObMicroBlockCSDecoder::filter_pushdown_filter( LOG_WARN("Failed to validate filter info", K(ret)); } else { int64_t col_count = filter.get_col_count(); + const ObColumnIndexArray &cols_index = read_info_->get_columns_index(); const common::ObIArray &col_offsets = filter.get_col_offsets(pd_filter_info.is_pd_to_cg_); const sql::ColumnParamFixedArray &col_params = filter.get_col_params(); decoder_allocator_.reuse(); @@ -1356,6 +1357,10 @@ int ObMicroBlockCSDecoder::filter_pushdown_filter( datum.reuse(); if (OB_FAIL(decoders_[col_offsets.at(i)].decode(row_idx, datum))) { LOG_WARN("decode cell failed", K(ret), K(row_idx), K(i), K(datum)); + } else if (OB_UNLIKELY(transform_helper_.get_micro_block_header()->is_trans_version_column_idx(cols_index.at(col_offsets.at(i))))) { + if (OB_FAIL(storage::reverse_trans_version_val(datum))) { + LOG_WARN("Failed to reverse trans version val", K(ret)); + } } else if (nullptr == col_params.at(i) || datum.is_null()) { } else if (col_params.at(i)->get_meta_type().is_fixed_len_char_type()) { if (OB_FAIL(storage::pad_column(col_params.at(i)->get_meta_type(), @@ -1785,6 +1790,13 @@ int ObMicroBlockCSDecoder::get_col_datums( LOG_WARN("fail to get datums from decoder", K(ret), K(col_id), K(row_cap), "row_ids", common::ObArrayWrap(row_ids, row_cap)); } + if (OB_SUCC(ret)) { + const ObColumnIndexArray &cols_index = read_info_->get_columns_index(); + if (OB_UNLIKELY(transform_helper_.get_micro_block_header()->is_trans_version_column_idx(cols_index.at(col_id)) && + OB_FAIL(storage::reverse_trans_version_val(col_datums, row_cap)))) { + LOG_WARN("Failed to reverse trans version val", K(ret)); + } + } return ret; } @@ -2007,11 +2019,15 @@ int ObMicroBlockCSDecoder::get_rows( int ObMicroBlockCSDecoder::get_col_data(const int32_t col_id, ObVectorDecodeCtx &vector_ctx) { int ret = OB_SUCCESS; + const ObColumnIndexArray &cols_index = read_info_->get_columns_index(); if (OB_UNLIKELY(col_id >= column_count_)) { ret = OB_INDEX_OUT_OF_RANGE; LOG_WARN("Vector store col id greate than store cnt", KR(ret), K(column_count_), K(col_id)); } else if (OB_FAIL(decoders_[col_id].decode_vector(vector_ctx))) { LOG_WARN("fail to get datums from decoder", K(ret), K(col_id), K(vector_ctx)); + } else if (OB_UNLIKELY(transform_helper_.get_micro_block_header()->is_trans_version_column_idx(cols_index.at(col_id))) && + OB_FAIL(storage::reverse_trans_version_val(vector_ctx.get_vector(), vector_ctx.row_cap_))) { + LOG_WARN("Failed to reverse trans version val", K(ret)); } return ret; } diff --git a/src/storage/blocksstable/encoding/ob_icolumn_decoder.h b/src/storage/blocksstable/encoding/ob_icolumn_decoder.h index 787abac3dd..e9398614a6 100644 --- a/src/storage/blocksstable/encoding/ob_icolumn_decoder.h +++ b/src/storage/blocksstable/encoding/ob_icolumn_decoder.h @@ -64,7 +64,8 @@ public: OB_INLINE void fill(const common::ObObjMeta &obj_meta, const ObMicroBlockHeader *micro_header, const ObColumnHeader *col_header, - common::ObIAllocator *allocator) + common::ObIAllocator *allocator, + const int64_t store_idx) { obj_meta_ = obj_meta; micro_block_header_ = micro_header; @@ -74,6 +75,7 @@ public: cache_attributes_[ObColumnHeader::FIX_LENGTH] = col_header_->is_fix_length(); cache_attributes_[ObColumnHeader::HAS_EXTEND_VALUE] = col_header_->has_extend_value(); cache_attributes_[ObColumnHeader::BIT_PACKING] = col_header_->is_bit_packing(); + cache_attributes_[ObColumnHeader::IS_TRANS_VERSION] = micro_block_header_->is_trans_version_column_idx(store_idx); } OB_INLINE bool is_fix_length() const { return cache_attributes_[ObColumnHeader::FIX_LENGTH]; } OB_INLINE bool has_extend_value() const { return cache_attributes_[ObColumnHeader::HAS_EXTEND_VALUE]; } @@ -82,6 +84,7 @@ public: { col_param_ = col_param; } + OB_INLINE bool is_trans_version_col() const { return cache_attributes_[ObColumnHeader::IS_TRANS_VERSION]; } TO_STRING_KV(K_(obj_meta), K_(micro_block_header), K_(col_header), KP_(allocator), KP_(ref_decoder), KP_(ref_ctx)); diff --git a/src/storage/blocksstable/encoding/ob_micro_block_decoder.cpp b/src/storage/blocksstable/encoding/ob_micro_block_decoder.cpp index 0b0c16a361..7c7855f49c 100644 --- a/src/storage/blocksstable/encoding/ob_micro_block_decoder.cpp +++ b/src/storage/blocksstable/encoding/ob_micro_block_decoder.cpp @@ -173,6 +173,9 @@ int ObColumnDecoder::batch_decode( } else if (OB_FAIL(decoder_->batch_decode( *ctx_, row_index, row_ids, cell_datas, row_cap, datums))) { LOG_WARN("Failed to batch decode data to datum in column decoder", K(ret), K(*ctx_)); + } else if (OB_UNLIKELY(ctx_->is_trans_version_col()) && + OB_FAIL(storage::reverse_trans_version_val(datums, row_cap))) { + LOG_WARN("Failed to reverse trans version val", K(ret)); } LOG_DEBUG("[Batch decode] Batch decoded datums: ", @@ -191,6 +194,9 @@ int ObColumnDecoder::decode_vector( LOG_WARN("Invalid null parameter", K(ret), KP(row_index), K(vector_ctx)); } else if (OB_FAIL(decoder_->decode_vector(*ctx_, row_index, vector_ctx))) { LOG_WARN("Failed to batch decode data to vector format in column decoder", K(ret), K(*ctx_)); + } else if (OB_UNLIKELY(ctx_->is_trans_version_col()) && + OB_FAIL(storage::reverse_trans_version_val(vector_ctx.get_vector(), vector_ctx.row_cap_))) { + LOG_WARN("Failed to reverse trans version val", K(ret)); } LOG_DEBUG("[Vector decode] Batch decoded datums: ", K(ret), K(*ctx_), K(vector_ctx)); return ret; @@ -557,7 +563,7 @@ int ObIEncodeBlockReader::add_decoder(const int64_t store_idx, const ObObjMeta & } else { dest.decoder_ = decoder; dest.ctx_ = ctxs_[store_idx]; - dest.ctx_->fill(obj_meta, header_, &col_header_[store_idx], &decoder_allocator_); + dest.ctx_->fill(obj_meta, header_, &col_header_[store_idx], &decoder_allocator_, store_idx); int64_t ref_col_idx = -1; if (NULL != cached_decocer_ && cached_decocer_->count_ > store_idx) { @@ -573,7 +579,7 @@ int ObIEncodeBlockReader::add_decoder(const int64_t store_idx, const ObObjMeta & } else { dest.ctx_->ref_decoder_ = ref_decoder; dest.ctx_->ref_ctx_ = ctxs_[ref_col_idx]; - dest.ctx_->ref_ctx_->fill(obj_meta, header_, &col_header_[ref_col_idx], &decoder_allocator_); + dest.ctx_->ref_ctx_->fill(obj_meta, header_, &col_header_[ref_col_idx], &decoder_allocator_, ref_col_idx); } } } @@ -1233,7 +1239,7 @@ int ObMicroBlockDecoder::add_decoder(const int64_t store_idx, const ObObjMeta &o } else { dest.decoder_ = decoder; dest.ctx_ = ctxs_[store_idx]; - dest.ctx_->fill(obj_meta, header_, &col_header_[store_idx], &decoder_allocator_.get_inner_allocator()); + dest.ctx_->fill(obj_meta, header_, &col_header_[store_idx], &decoder_allocator_.get_inner_allocator(), store_idx); int64_t ref_col_idx = -1; if (NULL != cached_decoder_ && cached_decoder_->count_ > store_idx) { @@ -1250,7 +1256,7 @@ int ObMicroBlockDecoder::add_decoder(const int64_t store_idx, const ObObjMeta &o } else { dest.ctx_->ref_decoder_ = decoder; dest.ctx_->ref_ctx_ = ctxs_[ref_col_idx]; - dest.ctx_->ref_ctx_->fill(obj_meta, header_, &col_header_[ref_col_idx], &decoder_allocator_.get_inner_allocator()); + dest.ctx_->ref_ctx_->fill(obj_meta, header_, &col_header_[ref_col_idx], &decoder_allocator_.get_inner_allocator(), ref_col_idx); } } } @@ -1714,6 +1720,7 @@ int ObMicroBlockDecoder::filter_pushdown_filter( LOG_WARN("Failed to validate filter info", K(ret)); } else { int64_t col_count = filter.get_col_count(); + const ObColumnIndexArray &cols_index = read_info_->get_columns_index(); const common::ObIArray &col_offsets = filter.get_col_offsets(pd_filter_info.is_pd_to_cg_); const sql::ColumnParamFixedArray &col_params = filter.get_col_params(); decoder_allocator_.reuse(); @@ -1735,6 +1742,10 @@ int ObMicroBlockDecoder::filter_pushdown_filter( datum.reuse(); if (OB_FAIL(decoders_[col_offsets.at(i)].decode(datum, row_idx, bs, row_data, row_len))) { LOG_WARN("decode cell failed", K(ret), K(row_idx), K(i), K(datum), K(bs), KP(row_data), K(row_len)); + } else if (OB_UNLIKELY(header_->is_trans_version_column_idx(cols_index.at(col_offsets.at(i))))) { + if (OB_FAIL(storage::reverse_trans_version_val(datum))) { + LOG_WARN("Failed to reverse trans version val", K(ret)); + } } else if (nullptr == col_params.at(i) || datum.is_null()) { } else if (col_params.at(i)->get_meta_type().is_fixed_len_char_type()) { if (OB_FAIL(storage::pad_column(col_params.at(i)->get_meta_type(), diff --git a/src/storage/blocksstable/ob_fuse_row_cache.cpp b/src/storage/blocksstable/ob_fuse_row_cache.cpp index 6ae509fdc8..c3b1347520 100644 --- a/src/storage/blocksstable/ob_fuse_row_cache.cpp +++ b/src/storage/blocksstable/ob_fuse_row_cache.cpp @@ -18,16 +18,15 @@ using namespace oceanbase::blocksstable; using namespace oceanbase::storage; -ObFuseRowCacheKey::ObFuseRowCacheKey() - : tenant_id_(0), rowkey_size_(0), rowkey_(), tablet_snapshot_version_(0), schema_column_count_(0), datum_utils_(nullptr) +ObFuseRowCacheKeyBase::ObFuseRowCacheKeyBase() + : tenant_id_(0), rowkey_size_(0), rowkey_(), schema_column_count_(0), datum_utils_(nullptr) { } -ObFuseRowCacheKey::ObFuseRowCacheKey( +ObFuseRowCacheKeyBase::ObFuseRowCacheKeyBase( const uint64_t tenant_id, const ObTabletID &tablet_id, const ObDatumRowkey &rowkey, - const int64_t tablet_snapshot_version, const int64_t schema_column_count, const ObStorageDatumUtils &datum_utils) { @@ -35,22 +34,15 @@ ObFuseRowCacheKey::ObFuseRowCacheKey( tablet_id_ = tablet_id; rowkey_ = rowkey; rowkey_size_ = rowkey.get_deep_copy_size(); - tablet_snapshot_version_ = tablet_snapshot_version; schema_column_count_ = schema_column_count; datum_utils_ = &datum_utils; } -uint64_t ObFuseRowCacheKey::get_tenant_id() const -{ - return tenant_id_; -} - -int ObFuseRowCacheKey::hash(uint64_t &hash_val) const +int ObFuseRowCacheKeyBase::hash(uint64_t &hash_val) const { int ret = OB_SUCCESS; hash_val = common::murmurhash(&tenant_id_, sizeof(tenant_id_), 0); hash_val = common::murmurhash(&tablet_id_, sizeof(tablet_id_), hash_val); - hash_val = common::murmurhash(&tablet_snapshot_version_, sizeof(tablet_snapshot_version_), hash_val); hash_val = common::murmurhash(&schema_column_count_, sizeof(schema_column_count_), hash_val); if (rowkey_.is_valid()) { if (OB_ISNULL(datum_utils_)) { @@ -63,30 +55,94 @@ int ObFuseRowCacheKey::hash(uint64_t &hash_val) const return ret; } -int ObFuseRowCacheKey::equal(const ObIKVCacheKey &other, bool &equal) const +int ObFuseRowCacheKeyBase::equal(const ObFuseRowCacheKeyBase &other, bool &equal) const { int ret = OB_SUCCESS; - const ObFuseRowCacheKey &other_key = reinterpret_cast(other); - equal = tenant_id_ == other_key.tenant_id_; - equal &= tablet_id_ == other_key.tablet_id_; - equal &= (rowkey_size_ == other_key.rowkey_size_); - equal &= tablet_snapshot_version_ == other_key.tablet_snapshot_version_; - equal &= schema_column_count_ == other_key.schema_column_count_; + equal = tenant_id_ == other.tenant_id_; + equal &= tablet_id_ == other.tablet_id_; + equal &= rowkey_size_ == other.rowkey_size_; + equal &= schema_column_count_ == other.schema_column_count_; if (equal && rowkey_size_ > 0) { - const ObStorageDatumUtils *datum_utils = (nullptr != datum_utils_) ? datum_utils_ : other_key.datum_utils_; + const ObStorageDatumUtils *datum_utils = (nullptr != datum_utils_) ? datum_utils_ : other.datum_utils_; if (OB_ISNULL(datum_utils)) { ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument to compare row cachekey", K(ret), K(*this), K(other_key)); - } else if (OB_FAIL(rowkey_.equal(other_key.rowkey_, *datum_utils, equal))) { - STORAGE_LOG(WARN, "Failed to check rowkey cache key equal", K(ret), K(rowkey_), K(other_key)); + STORAGE_LOG(WARN, "Invalid argument to compare row cachekey", K(ret), K(*this), K(other)); + } else if (OB_FAIL(rowkey_.equal(other.rowkey_, *datum_utils, equal))) { + STORAGE_LOG(WARN, "Failed to check rowkey cache key equal", K(ret), K(rowkey_), K(other)); } } return ret; } +int ObFuseRowCacheKeyBase::deep_copy(char *buf, const int64_t buf_len, ObFuseRowCacheKeyBase &dest) const +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(nullptr == buf || buf_len < rowkey_size_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), KP(buf), K(buf_len), "request_size", rowkey_size_); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_INVALID_DATA; + LOG_WARN("invalid fuse row cache key", K(ret), K(*this)); + } else { + dest.tenant_id_ = tenant_id_; + dest.tablet_id_ = tablet_id_; + dest.schema_column_count_ = schema_column_count_; + if (rowkey_.is_valid() && rowkey_size_ > 0) { + ObRawBufAllocatorWrapper tmp_buf(buf, rowkey_size_); + if (OB_FAIL(rowkey_.deep_copy(dest.rowkey_, tmp_buf))) { + LOG_WARN("fail to deep copy rowkey", K(ret)); + } else { + dest.rowkey_size_ = rowkey_size_; + } + } + } + return ret; +} + +ObFuseRowCacheKey::ObFuseRowCacheKey() + : base_(), + tablet_snapshot_version_(0) +{ +} + +ObFuseRowCacheKey::ObFuseRowCacheKey( + const uint64_t tenant_id, + const ObTabletID &tablet_id, + const ObDatumRowkey &rowkey, + const int64_t tablet_snapshot_version, + const int64_t schema_column_count, + const ObStorageDatumUtils &datum_utils) + : base_(tenant_id, tablet_id, rowkey, schema_column_count, datum_utils), + tablet_snapshot_version_(tablet_snapshot_version) +{ +} + +int ObFuseRowCacheKey::equal(const ObIKVCacheKey &other, bool &equal) const +{ + int ret = OB_SUCCESS; + const ObFuseRowCacheKey &other_key = reinterpret_cast(other); + return tablet_snapshot_version_ == other_key.tablet_snapshot_version_ && base_.equal(other_key.base_, equal); +} + +int ObFuseRowCacheKey::hash(uint64_t &hash_value) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(base_.hash(hash_value))) { + LOG_WARN("Failed to hash base key", K(ret), K(*this)); + } else { + hash_value = common::murmurhash(&tablet_snapshot_version_, sizeof(tablet_snapshot_version_), hash_value); + } + return ret; +} + +uint64_t ObFuseRowCacheKey::get_tenant_id() const +{ + return base_.get_tenant_id(); +} + int64_t ObFuseRowCacheKey::size() const { - return sizeof(*this) + rowkey_size_; + return sizeof(*this) + base_.rowkey_size(); } int ObFuseRowCacheKey::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey *&key) const @@ -100,18 +156,11 @@ int ObFuseRowCacheKey::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey LOG_WARN("invalid fuse row cache key", K(ret), K(*this)); } else { ObFuseRowCacheKey *pfuse_key = new (buf) ObFuseRowCacheKey(); - pfuse_key->tenant_id_ = tenant_id_; - pfuse_key->tablet_id_ = tablet_id_; pfuse_key->tablet_snapshot_version_ = tablet_snapshot_version_; - pfuse_key->schema_column_count_ = schema_column_count_; - if (rowkey_.is_valid() && rowkey_size_ > 0) { - ObRawBufAllocatorWrapper tmp_buf(buf + sizeof(*this), rowkey_size_); - if (OB_FAIL(rowkey_.deep_copy(pfuse_key->rowkey_, tmp_buf))) { - LOG_WARN("fail to deep copy rowkey", K(ret)); - } else { - pfuse_key->rowkey_size_ = rowkey_size_; - key = pfuse_key; - } + if (OB_FAIL(base_.deep_copy(buf + sizeof(ObFuseRowCacheKey), buf_len - sizeof(ObFuseRowCacheKey), pfuse_key->base_))) { + LOG_WARN("fail to deep copy base key", K(ret)); + } else { + key = pfuse_key; } if (OB_FAIL(ret)) { pfuse_key->~ObFuseRowCacheKey(); @@ -123,7 +172,7 @@ int ObFuseRowCacheKey::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey bool ObFuseRowCacheKey::is_valid() const { - return OB_LIKELY(tenant_id_ != 0 && tablet_id_.is_valid() && rowkey_size_ > 0 && rowkey_.is_valid() && tablet_snapshot_version_ >= 0 && schema_column_count_ >= 0); + return OB_LIKELY(base_.is_valid() && tablet_snapshot_version_ >= 0); } ObFuseRowCacheValue::ObFuseRowCacheValue() @@ -230,3 +279,122 @@ int ObFuseRowCache::put_row(const ObFuseRowCacheKey &key, const ObFuseRowCacheVa } return ret; } + +ObMultiVersionFuseRowCacheKey::ObMultiVersionFuseRowCacheKey() + : base_(), begin_version_(0), end_version_(0) +{ +} + +ObMultiVersionFuseRowCacheKey::ObMultiVersionFuseRowCacheKey( + const int64_t begin_version, + const int64_t end_version, + const uint64_t tenant_id, + const ObTabletID &tablet_id, + const ObDatumRowkey &rowkey, + const int64_t schema_column_count, + const ObStorageDatumUtils &datum_utils) + : base_(tenant_id, tablet_id, rowkey, schema_column_count, datum_utils), + begin_version_(begin_version), + end_version_(end_version) +{ +} + +int ObMultiVersionFuseRowCacheKey::equal(const ObIKVCacheKey &other, bool &equal) const +{ + int ret = OB_SUCCESS; + const ObMultiVersionFuseRowCacheKey &other_key = reinterpret_cast(other); + equal = begin_version_ == other_key.begin_version_ && + end_version_ == other_key.end_version_; + if (equal && OB_FAIL(base_.equal(other_key.base_, equal))) { + LOG_WARN("Failed to check if fuse row keys are equal", K(ret), K(*this), K(other_key)); + } + return ret; +} + +int ObMultiVersionFuseRowCacheKey::hash(uint64_t &hash_value) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(base_.hash(hash_value))) { + LOG_WARN("Failed to hash base key", K(ret), K(*this)); + } else { + hash_value = common::murmurhash(&begin_version_, sizeof(begin_version_), hash_value); + hash_value = common::murmurhash(&end_version_, sizeof(end_version_), hash_value); + } + return ret; +} + +uint64_t ObMultiVersionFuseRowCacheKey::get_tenant_id() const +{ + return base_.get_tenant_id(); +} + +int64_t ObMultiVersionFuseRowCacheKey::size() const +{ + return sizeof(*this) + base_.rowkey_size(); +} + +int ObMultiVersionFuseRowCacheKey::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey *&key) const +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(nullptr == buf || buf_len < size())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), KP(buf), K(buf_len), "request_size", size()); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_INVALID_DATA; + LOG_WARN("invalid multi version fuse row cache key", K(ret), K(*this)); + } else { + ObMultiVersionFuseRowCacheKey *pfuse_key = new (buf) ObMultiVersionFuseRowCacheKey(); + pfuse_key->begin_version_ = begin_version_; + pfuse_key->end_version_ = end_version_; + if (OB_FAIL(base_.deep_copy(buf + sizeof(ObMultiVersionFuseRowCacheKey), buf_len - sizeof(ObMultiVersionFuseRowCacheKey), pfuse_key->base_))) { + LOG_WARN("fail to deep copy base key", K(ret)); + } else { + key = pfuse_key; + } + if (OB_FAIL(ret)) { + pfuse_key->~ObMultiVersionFuseRowCacheKey(); + pfuse_key = nullptr; + } + } + return ret; +} + +bool ObMultiVersionFuseRowCacheKey::is_valid() const +{ + return OB_LIKELY(begin_version_ >= 0 && end_version_ > begin_version_ && base_.is_valid()); +} + +int ObMultiVersionFuseRowCache::get_row(const ObMultiVersionFuseRowCacheKey &key, ObFuseRowValueHandle &handle) +{ + int ret = OB_SUCCESS; + const ObFuseRowCacheValue *value = nullptr; + if (OB_UNLIKELY(!key.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), K(key)); + } else if (OB_FAIL(get(key, value, handle.handle_))) { + if (OB_UNLIKELY(OB_ENTRY_NOT_EXIST != ret)) { + LOG_WARN("fail to get key from row cache", K(ret)); + } + EVENT_INC(ObStatEventIds::MULTI_VERSION_FUSE_ROW_CACHE_MISS); + } else { + if (OB_ISNULL(value)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error, the value must not be NULL", K(ret)); + } else { + handle.value_ = const_cast(value); + } + } + return ret; +} + +int ObMultiVersionFuseRowCache::put_row(const ObMultiVersionFuseRowCacheKey &key, const ObFuseRowCacheValue &value) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!key.is_valid() || !value.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), K(key), K(value)); + } else if (OB_FAIL(put(key, value, true/*overwrite*/))) { + LOG_WARN("fail to put row to row cache", K(ret), K(key), K(value)); + } + return ret; +} \ No newline at end of file diff --git a/src/storage/blocksstable/ob_fuse_row_cache.h b/src/storage/blocksstable/ob_fuse_row_cache.h index a094437e05..6511e5114c 100644 --- a/src/storage/blocksstable/ob_fuse_row_cache.h +++ b/src/storage/blocksstable/ob_fuse_row_cache.h @@ -22,6 +22,36 @@ namespace oceanbase namespace blocksstable { +struct ObFuseRowCacheKeyBase final +{ +public: + ObFuseRowCacheKeyBase(); + ObFuseRowCacheKeyBase( + const uint64_t tenant_id, + const ObTabletID &tablet_id, + const ObDatumRowkey &rowkey, + const int64_t schema_column_count, + const ObStorageDatumUtils &datum_utils); + int hash(uint64_t &hash_value) const; + int equal(const ObFuseRowCacheKeyBase &other, bool &equal) const; + int deep_copy(char *buf, const int64_t buf_len, ObFuseRowCacheKeyBase &dest) const; + OB_INLINE bool is_valid() const + { + return OB_LIKELY(tenant_id_ != 0 && tablet_id_.is_valid() && rowkey_size_ > 0 && rowkey_.is_valid() && schema_column_count_ >= 0); + } + OB_INLINE int64_t rowkey_size() const { return rowkey_size_; } + OB_INLINE uint64_t get_tenant_id() const { return tenant_id_; } + TO_STRING_KV(K_(tenant_id), K_(tablet_id), K_(rowkey_size), K_(rowkey), K_(schema_column_count), KPC_(datum_utils)); +private: + uint64_t tenant_id_; + ObTabletID tablet_id_; + int64_t rowkey_size_; + ObDatumRowkey rowkey_; + int64_t schema_column_count_; + const ObStorageDatumUtils *datum_utils_; + DISALLOW_COPY_AND_ASSIGN(ObFuseRowCacheKeyBase); +}; + class ObFuseRowCacheKey : public common::ObIKVCacheKey { public: @@ -40,15 +70,10 @@ public: virtual int64_t size() const override; virtual int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey *&key) const override; bool is_valid() const; - TO_STRING_KV(K_(tenant_id), K_(tablet_id), K_(rowkey_size), K_(rowkey), K_(tablet_snapshot_version), K_(schema_column_count), KPC_(datum_utils)); + TO_STRING_KV(K_(base)); private: - uint64_t tenant_id_; - ObTabletID tablet_id_; - int64_t rowkey_size_; - ObDatumRowkey rowkey_; + ObFuseRowCacheKeyBase base_; int64_t tablet_snapshot_version_; - int64_t schema_column_count_; - const ObStorageDatumUtils *datum_utils_; DISALLOW_COPY_AND_ASSIGN(ObFuseRowCacheKey); }; @@ -102,6 +127,45 @@ private: DISALLOW_COPY_AND_ASSIGN(ObFuseRowCache); }; +class ObMultiVersionFuseRowCacheKey : public common::ObIKVCacheKey +{ +public: + ObMultiVersionFuseRowCacheKey(); + ObMultiVersionFuseRowCacheKey( + const int64_t begin_version, + const int64_t end_version, + const uint64_t tenant_id, + const ObTabletID &tablet_id, + const ObDatumRowkey &rowkey, + const int64_t schema_column_count, + const ObStorageDatumUtils &datum_utils); + virtual ~ObMultiVersionFuseRowCacheKey() = default; + virtual int equal(const ObIKVCacheKey &other, bool &equal) const override; + virtual int hash(uint64_t &hash_value) const override; + virtual uint64_t get_tenant_id() const override; + virtual int64_t size() const override; + virtual int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey *&key) const override; + bool is_valid() const; + TO_STRING_KV(K_(base), K_(begin_version), K_(end_version)); +private: + ObFuseRowCacheKeyBase base_; + // (begin_version_, end_version] + int64_t begin_version_; + int64_t end_version_; + DISALLOW_COPY_AND_ASSIGN(ObMultiVersionFuseRowCacheKey); +}; + +class ObMultiVersionFuseRowCache : public common::ObKVCache +{ +public: + ObMultiVersionFuseRowCache() = default; + virtual ~ObMultiVersionFuseRowCache() = default; + int get_row(const ObMultiVersionFuseRowCacheKey &key, ObFuseRowValueHandle &handle); + int put_row(const ObMultiVersionFuseRowCacheKey &key, const ObFuseRowCacheValue &value); +private: + DISALLOW_COPY_AND_ASSIGN(ObMultiVersionFuseRowCache); +}; + } // end namespace storage } // end namespace oceanbase diff --git a/src/storage/blocksstable/ob_micro_block_header.h b/src/storage/blocksstable/ob_micro_block_header.h index d52ff9a109..788a402902 100644 --- a/src/storage/blocksstable/ob_micro_block_header.h +++ b/src/storage/blocksstable/ob_micro_block_header.h @@ -115,6 +115,8 @@ public: OB_INLINE bool has_lob_out_row() const { return !all_lob_in_row_; } bool is_last_row_last_flag() const { return is_last_row_last_flag_; } bool is_contain_hash_index() const; + bool is_trans_version_column_idx(const int64_t col_idx) const + { return rowkey_column_count_ > 0 && col_idx == rowkey_column_count_ - storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); } }__attribute__((packed)); }//end namespace blocksstable }//end namespace oceanbase diff --git a/src/storage/blocksstable/ob_micro_block_reader.cpp b/src/storage/blocksstable/ob_micro_block_reader.cpp index 247629eea7..7ba493503a 100644 --- a/src/storage/blocksstable/ob_micro_block_reader.cpp +++ b/src/storage/blocksstable/ob_micro_block_reader.cpp @@ -726,6 +726,8 @@ int ObMicroBlockReader::filter_pushdown_filter( col_idx, tmp_datum))) { LOG_WARN("fail to read column", K(ret), K(i), K(col_idx), K(row_idx), KPC_(header)); + } else if (OB_UNLIKELY(header_->is_trans_version_column_idx(col_idx))) { + datum.set_int(-tmp_datum.get_int()); } else if (tmp_datum.is_nop_value()) { if (OB_UNLIKELY(default_datums.at(i).is_nop())) { ret = OB_ERR_UNEXPECTED; @@ -814,6 +816,7 @@ int ObMicroBlockReader::get_rows( } else if (OB_FAIL(row_buf.reserve(read_info_->get_request_count()))) { LOG_WARN("Failed to reserve row buf", K(ret), K(row_buf), KPC(read_info_)); } else { + const ObColumnIndexArray &cols_index = read_info_->get_columns_index(); for (int64_t idx = 0; OB_SUCC(ret) && idx < row_cap; ++idx) { row_idx = row_ids[idx]; if (OB_UNLIKELY(row_idx < 0 || row_idx >= header_->row_count_)) { @@ -832,6 +835,8 @@ int ObMicroBlockReader::get_rows( if (col_idx >= read_info_->get_request_count()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected col idx", K(ret), K(i), K(col_idx), K(read_info_->get_request_count())); + } else if (OB_UNLIKELY(header_->is_trans_version_column_idx(cols_index.at(col_idx)))) { + datum.set_int(-row_buf.storage_datums_[col_idx].get_int()); } else if (row_buf.storage_datums_[col_idx].is_null()) { datum.set_null(); } else if (row_buf.storage_datums_[col_idx].is_nop()) { @@ -913,6 +918,8 @@ int ObMicroBlockReader::get_rows( } } if (OB_SUCC(ret)) { + ObStorageDatum trans_datum; + const ObColumnIndexArray &cols_index = read_info_->get_columns_index(); for (int64_t idx = 0; OB_SUCC(ret) && idx < row_cap; ++idx) { row_idx = row_ids[idx]; if (OB_UNLIKELY(row_idx < 0 || row_idx >= header_->row_count_)) { @@ -932,6 +939,10 @@ int ObMicroBlockReader::get_rows( if (col_idx >= read_info_->get_request_count()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected col idx", K(ret), K(i), K(col_idx), K(read_info_->get_request_count())); + } else if (OB_UNLIKELY(header_->is_trans_version_column_idx(cols_index.at(col_idx)))) { + trans_datum.reuse(); + trans_datum.set_int(-row_buf.storage_datums_[col_idx].get_int()); + col_datum = &trans_datum; } else if (row_buf.storage_datums_[col_idx].is_nop()) { if (OB_ISNULL(default_row)) { ret = OB_ERR_UNEXPECTED; diff --git a/src/storage/blocksstable/ob_micro_block_row_scanner.cpp b/src/storage/blocksstable/ob_micro_block_row_scanner.cpp index 71f7d64aee..af906efa7f 100644 --- a/src/storage/blocksstable/ob_micro_block_row_scanner.cpp +++ b/src/storage/blocksstable/ob_micro_block_row_scanner.cpp @@ -555,8 +555,9 @@ int ObIMicroBlockRowScanner::apply_black_filter_batch( const common::ObIArray &col_offsets = filter.get_col_offsets(pd_filter_info.is_pd_to_cg_); ObSEArray datum_infos; bool filter_applied_directly = false; - if (ObIMicroBlockReader::Decoder == reader_->get_type() || - ObIMicroBlockReader::CSDecoder == reader_->get_type()) { + if ((ObIMicroBlockReader::Decoder == reader_->get_type() || + ObIMicroBlockReader::CSDecoder == reader_->get_type()) && + filter.can_pushdown_decoder()) { if (decoder_->can_apply_black(col_offsets) && OB_FAIL(decoder_->filter_black_filter_batch( parent, @@ -1915,7 +1916,7 @@ int ObMultiVersionMicroBlockRowScanner::lock_for_read( if (OB_FAIL(tx_table_guards.lock_for_read(lock_for_read_arg, can_read, scn_trans_version))) { - LOG_WARN("failed to check transaction status", K(ret)); + LOG_WARN("failed to check transaction status", K(ret), K(*context_->store_ctx_)); } else { trans_version = scn_trans_version.get_val_for_tx(); if (OB_NOT_NULL(context_->trans_state_mgr_) && diff --git a/src/storage/blocksstable/ob_storage_cache_suite.cpp b/src/storage/blocksstable/ob_storage_cache_suite.cpp index d56992ea57..b48f0854af 100644 --- a/src/storage/blocksstable/ob_storage_cache_suite.cpp +++ b/src/storage/blocksstable/ob_storage_cache_suite.cpp @@ -25,6 +25,7 @@ ObStorageCacheSuite::ObStorageCacheSuite() bf_cache_(), fuse_row_cache_(), storage_meta_cache_(), + multi_version_fuse_row_cache_(), is_inited_(false) { } @@ -67,6 +68,8 @@ int ObStorageCacheSuite::init( STORAGE_LOG(ERROR, "fail to init fuse row cache", K(ret)); } else if (OB_FAIL(storage_meta_cache_.init("storage_meta_cache", storage_meta_cache_priority))) { STORAGE_LOG(ERROR, "fail to init storage meta cache", K(ret), K(storage_meta_cache_priority)); + } else if (OB_FAIL(multi_version_fuse_row_cache_.init("multi_version_fuse_row_cache", fuse_row_cache_priority))) { + STORAGE_LOG(ERROR, "fail to init multi version fuse row cache", K(ret)); } else { is_inited_ = true; } @@ -102,6 +105,8 @@ int ObStorageCacheSuite::reset_priority( STORAGE_LOG(ERROR, "fail to set priority for fuse row cache", K(ret)); } else if (OB_FAIL(storage_meta_cache_.set_priority(storage_meta_cache_priority))) { STORAGE_LOG(ERROR, "fail to set priority for storage cache", K(ret), K(storage_meta_cache_priority)); + } else if (OB_FAIL(multi_version_fuse_row_cache_.set_priority(fuse_row_cache_priority))) { + STORAGE_LOG(ERROR, "fail to set priority for multi version fuse row cache", K(ret)); } return ret; } @@ -123,6 +128,7 @@ void ObStorageCacheSuite::destroy() bf_cache_.destroy(); fuse_row_cache_.destroy(); storage_meta_cache_.destory(); + multi_version_fuse_row_cache_.destroy(); is_inited_ = false; } diff --git a/src/storage/blocksstable/ob_storage_cache_suite.h b/src/storage/blocksstable/ob_storage_cache_suite.h index ad9d0b6535..17a3fc553d 100644 --- a/src/storage/blocksstable/ob_storage_cache_suite.h +++ b/src/storage/blocksstable/ob_storage_cache_suite.h @@ -54,6 +54,7 @@ public: ObRowCache &get_row_cache() { return user_row_cache_; } ObBloomFilterCache &get_bf_cache() { return bf_cache_; } ObFuseRowCache &get_fuse_row_cache() { return fuse_row_cache_; } + ObMultiVersionFuseRowCache &get_multi_version_fuse_row_cache() { return multi_version_fuse_row_cache_; } ObStorageMetaCache &get_storage_meta_cache() { return storage_meta_cache_; } void destroy(); inline bool is_inited() const { return is_inited_; } @@ -67,6 +68,7 @@ private: ObBloomFilterCache bf_cache_; ObFuseRowCache fuse_row_cache_; ObStorageMetaCache storage_meta_cache_; + ObMultiVersionFuseRowCache multi_version_fuse_row_cache_; bool is_inited_; private: DISALLOW_COPY_AND_ASSIGN(ObStorageCacheSuite); diff --git a/src/storage/column_store/ob_cg_sstable_row_getter.cpp b/src/storage/column_store/ob_cg_sstable_row_getter.cpp index a544c36c68..99325be67a 100644 --- a/src/storage/column_store/ob_cg_sstable_row_getter.cpp +++ b/src/storage/column_store/ob_cg_sstable_row_getter.cpp @@ -416,7 +416,7 @@ int ObCGSSTableRowGetter::fetch_row(ObSSTableReadHandle &read_handle, const ObNo if (OB_SUCC(ret) && iter_param_->need_scn_) { if (OB_FAIL(fetch_rowkey_row(read_handle, store_row))) { LOG_WARN("Fail to fetch row", K(ret)); - } else if (OB_FAIL(set_row_scn(*iter_param_, store_row))) { + } else if (OB_FAIL(set_row_scn(access_ctx_->use_fuse_row_cache_, *iter_param_, store_row))) { LOG_WARN("failed to set row scn", K(ret)); } else { int64_t trans_idx = iter_param_->get_read_info()->get_trans_col_index(); diff --git a/src/storage/column_store/ob_co_sstable_row_scanner.h b/src/storage/column_store/ob_co_sstable_row_scanner.h index 3b09ee4b82..a50849ccfd 100644 --- a/src/storage/column_store/ob_co_sstable_row_scanner.h +++ b/src/storage/column_store/ob_co_sstable_row_scanner.h @@ -56,6 +56,7 @@ public: virtual bool can_batch_scan() const override { return can_blockscan() && + !access_ctx_->is_mview_query() && iter_param_->vectorized_enabled_ && iter_param_->enable_pd_filter(); } diff --git a/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp b/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp index 14be6ab6bb..bf34731b4d 100644 --- a/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp +++ b/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp @@ -788,6 +788,9 @@ void ObBasicTabletMergeCtx::add_sstable_merge_info( ADD_COMMENT("cost_mb", mem_peak_mb); } ADD_COMMENT("time", time_guard); + if (nullptr != static_param_.schema_ && static_param_.schema_->is_mv_major_refresh_table()) { + ADD_COMMENT("mv", 1); + } #undef ADD_COMMENT ObInfoParamBuffer info_allocator; diff --git a/src/storage/compaction/ob_compaction_diagnose.cpp b/src/storage/compaction/ob_compaction_diagnose.cpp index f969833691..f8230e912a 100644 --- a/src/storage/compaction/ob_compaction_diagnose.cpp +++ b/src/storage/compaction/ob_compaction_diagnose.cpp @@ -1299,10 +1299,16 @@ int ObCompactionDiagnoseMgr::diagnose_tablet_major_merge( const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; const int64_t last_major_snapshot_version = tablet.get_last_major_snapshot_version(); int64_t max_sync_medium_scn = 0; + ObArenaAllocator temp_allocator("GetSSchema", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); + ObStorageSchema *storage_schema = nullptr; + bool is_mv_major_refresh_tablet = false; if (tablet_id.is_ls_inner_tablet()) { // do nothing } else if (OB_FAIL(tablet.get_max_sync_medium_scn(max_sync_medium_scn))){ LOG_WARN("failed to get max sync medium scn", K(ret), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(tablet.load_storage_schema(temp_allocator, storage_schema))) { + LOG_WARN("failed to load storage schema", K(ret), K(tablet)); + } else if (FALSE_IT(is_mv_major_refresh_tablet = storage_schema->is_mv_major_refresh())) { } else { LOG_TRACE("diagnose tablet major merge", K(ls_id), K(tablet_id), K(compaction_scn), K(max_sync_medium_scn), K(last_major_snapshot_version)); if (last_major_snapshot_version < compaction_scn) { @@ -1325,6 +1331,13 @@ int ObCompactionDiagnoseMgr::diagnose_tablet_major_merge( } else if (0 == last_major_snapshot_version) { const char* info = "no major sstable"; ADD_MAJOR_WAIT_SCHEDULE(compaction_scn + WAIT_MEDIUM_SCHEDULE_INTERVAL, info); + } else if (is_mv_major_refresh_tablet && + ObBasicMergeScheduler::INIT_COMPACTION_SCN == last_major_snapshot_version) { + if (OB_FAIL(ADD_DIAGNOSE_INFO_FOR_TABLET( + MEDIUM_MERGE, ObCompactionDiagnoseInfo::DIA_STATUS_NOT_SCHEDULE, + ObTimeUtility::fast_current_time(), "current_status", "wait for materialized view creation"))) { + LOG_WARN("failed to add diagnose info", K(ret), K(ls_id), K(tablet_id)); + } } if (OB_TMP_FAIL(diagnose_tablet_merge( MEDIUM_MERGE, diff --git a/src/storage/compaction/ob_medium_compaction_func.cpp b/src/storage/compaction/ob_medium_compaction_func.cpp index 342439fcec..7ea82acf22 100644 --- a/src/storage/compaction/ob_medium_compaction_func.cpp +++ b/src/storage/compaction/ob_medium_compaction_func.cpp @@ -1159,6 +1159,9 @@ int ObMediumCompactionScheduleFunc::submit_medium_clog( if (OB_UNLIKELY(!tablet_handle_.is_valid())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid tablet_handle", K(ret), K(tablet_handle_)); + } else if (OB_UNLIKELY(medium_info.is_invalid_mview_compaction())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid mview compaction", K(ret), K(tablet_handle_), K(medium_info)); } else if (FALSE_IT(tablet = tablet_handle_.get_obj())) { } else if (OB_FAIL(tablet->submit_medium_compaction_clog(medium_info, allocator_))) { LOG_WARN("failed to submit medium compaction clog", K(ret), K(medium_info)); @@ -1487,17 +1490,36 @@ int ObMediumCompactionScheduleFunc::schedule_tablet_medium_merge( const ObLSID &ls_id = ls.get_ls_id(); ObArenaAllocator temp_allocator("GetMediumInfo", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); // for load medium info const ObMediumCompactionInfoList *medium_list = nullptr; + ObStorageSchema *storage_schema = nullptr; bool schedule_flag = false; const int64_t major_frozen_snapshot = 0 == input_major_snapshot ? MTL(ObTenantTabletScheduler *)->get_frozen_version() : input_major_snapshot; // broadcast scn ObMediumCompactionInfo::ObCompactionType compaction_type = ObMediumCompactionInfo::COMPACTION_TYPE_MAX; int64_t schedule_scn = 0; // medium_snapshot in medium info bool tablet_need_freeze_flag = false; - - if (OB_FAIL(tablet.read_medium_info_list(temp_allocator, medium_list))) { + bool is_mv_major_refresh_tablet = false; + // temp solution(load storage schema to decide whether tablet is MV) + // TODO replace with new func on tablet later @lana + if (OB_FAIL(tablet.load_storage_schema(temp_allocator, storage_schema))) { + LOG_WARN("failed to load storage schema", K(ret), K(tablet)); + } else if (FALSE_IT(is_mv_major_refresh_tablet = storage_schema->is_mv_major_refresh())) { + } else if (is_mv_major_refresh_tablet && + ObBasicMergeScheduler::INIT_COMPACTION_SCN == last_major_snapshot) { + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(ADD_SUSPECT_INFO( + MEDIUM_MERGE, share::ObDiagnoseTabletType::TYPE_MEDIUM_MERGE, ls_id, tablet_id, + ObSuspectInfoType::SUSPECT_MV_IN_CREATION, input_major_snapshot, + static_cast(tablet.is_row_store())))) { + LOG_WARN("failed to add suspect info", K(tmp_ret)); + } + LOG_INFO("mv creation has not finished, can not schedule mv tablet", K(ret), K(last_major_snapshot)); + } else if (OB_FAIL(tablet.read_medium_info_list(temp_allocator, medium_list))) { LOG_WARN("failed to load medium info list", K(ret), K(tablet)); - } else if (OB_FAIL(read_medium_info_from_list(*medium_list, last_major_snapshot, major_frozen_snapshot, compaction_type, schedule_scn))) { + } else if (OB_FAIL(read_medium_info_from_list( + *medium_list, last_major_snapshot, major_frozen_snapshot, + is_mv_major_refresh_tablet, compaction_type, schedule_scn))) { LOG_WARN("failed to read medium info from list", K(ret), K(ls_id), K(tablet_id), KPC(medium_list), K(last_major_snapshot)); - } else if (is_standy_tenant) { // for STANDBY/RESTORE TENANT + } else if (is_standy_tenant && !is_mv_major_refresh_tablet) { // for STANDBY/RESTORE TENANT + // for mv tablet, schedule exist medium info without checking major_frozen_snapshot if (OB_FAIL(decide_standy_tenant_schedule(ls_id, tablet_id, compaction_type, schedule_scn, major_frozen_snapshot, *medium_list, schedule_flag))) { LOG_WARN("failed to decide whehter to schedule standy schedule", K(ret), K(ls_id), K(tablet_id), K(compaction_type), K(schedule_scn), K(major_frozen_snapshot), K(medium_list)); } @@ -1532,6 +1554,7 @@ int ObMediumCompactionScheduleFunc::decide_standy_tenant_schedule( { int ret = OB_SUCCESS; schedule_flag = false; + if (ObMediumCompactionInfo::MAJOR_COMPACTION == compaction_type) { if (schedule_scn > major_frozen_snapshot) { ret = OB_ERR_UNEXPECTED; @@ -1561,6 +1584,7 @@ int ObMediumCompactionScheduleFunc::read_medium_info_from_list( const ObMediumCompactionInfoList &medium_list, const int64_t last_major_snapshot, const int64_t major_frozen_snapshot, + const bool is_mv_refresh_tablet, ObMediumCompactionInfo::ObCompactionType &compaction_type, int64_t &schedule_scn) { @@ -1570,7 +1594,8 @@ int ObMediumCompactionScheduleFunc::read_medium_info_from_list( // finished, this medium info could recycle } else { if (info->is_medium_compaction() - || info->medium_snapshot_ <= major_frozen_snapshot) { + || info->medium_snapshot_ <= major_frozen_snapshot + || is_mv_refresh_tablet) { schedule_scn = info->medium_snapshot_; compaction_type = (ObMediumCompactionInfo::ObCompactionType)info->compaction_type_; } diff --git a/src/storage/compaction/ob_medium_compaction_func.h b/src/storage/compaction/ob_medium_compaction_func.h index b825dc9d99..a300a0989e 100644 --- a/src/storage/compaction/ob_medium_compaction_func.h +++ b/src/storage/compaction/ob_medium_compaction_func.h @@ -74,6 +74,7 @@ public: const ObMediumCompactionInfoList &medium_list, const int64_t major_frozen_snapshot, const int64_t last_major_snapshot, + const bool is_mv_refresh_tablet, ObMediumCompactionInfo::ObCompactionType &compaction_type, int64_t &schedule_scn); static int is_election_leader(const share::ObLSID &ls_id, bool &ls_election_leader); diff --git a/src/storage/compaction/ob_medium_compaction_info.h b/src/storage/compaction/ob_medium_compaction_info.h index 1de94a25ad..ac92d47248 100644 --- a/src/storage/compaction/ob_medium_compaction_info.h +++ b/src/storage/compaction/ob_medium_compaction_info.h @@ -219,6 +219,7 @@ public: static inline bool is_major_compaction(const ObCompactionType type) { return MAJOR_COMPACTION == type; } inline bool is_major_compaction() const { return is_major_compaction((ObCompactionType)compaction_type_); } inline bool is_medium_compaction() const { return is_medium_compaction((ObCompactionType)compaction_type_); } + inline bool is_invalid_mview_compaction() const { return storage_schema_.is_mv_major_refresh_table() && medium_merge_reason_ != ObAdaptiveMergePolicy::TENANT_MAJOR; } void clear_parallel_range() { parallel_merge_info_.clear(); diff --git a/src/storage/compaction/ob_mview_compaction_util.cpp b/src/storage/compaction/ob_mview_compaction_util.cpp new file mode 100644 index 0000000000..c007cfc50e --- /dev/null +++ b/src/storage/compaction/ob_mview_compaction_util.cpp @@ -0,0 +1,390 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE +#include "ob_mview_compaction_util.h" +#include "ob_tablet_merge_task.h" +#include "ob_basic_tablet_merge_ctx.h" +#include "observer/ob_server_struct.h" +#include "sql/session/ob_sql_session_info.h" +#include "sql/session/ob_sql_session_mgr.h" +#include "sql/resolver/mv/ob_mv_provider.h" +#include "observer/ob_inner_sql_connection_pool.h" + +namespace oceanbase +{ +namespace compaction +{ + +ObMviewMergeParameter::ObMviewMergeParameter() + : database_id_(0), + mview_id_(0), + container_table_id_(0), + container_tablet_id_(), + schema_version_(0), + refresh_scn_range_(), + refresh_sql_count_(0) +{ +} + +ObMviewMergeParameter::~ObMviewMergeParameter() +{ +} + +int ObMviewMergeParameter::init(const ObMergeParameter &merge_param) +{ + int ret = OB_SUCCESS; + refresh_sql_count_ = REFRESH_SQL_COUNT_V1; + container_tablet_id_ = merge_param.static_param_.get_tablet_id(); + schema_version_ = merge_param.get_schema()->get_schema_version(); + ObSEArray tablet_ids; + ObSEArray table_ids; + ObSchemaGetterGuard schema_guard; + sql::ObSQLSessionInfo *session = nullptr; + const ObTableSchema *table_schema = nullptr; + const uint64_t tenant_id = MTL_ID(); + sql::ObFreeSessionCtx free_session_ctx; + if (OB_FAIL(tablet_ids.push_back(container_tablet_id_))) { + LOG_WARN("Failed to add tablet id", K(ret), K_(container_tablet_id)); + } else if (OB_FAIL(GCTX.schema_service_->get_tablet_to_table_history(tenant_id, tablet_ids, schema_version_, table_ids))) { + LOG_WARN("Failed to get table id according to tablet id", K(ret), K_(container_tablet_id), K_(schema_version)); + } else if (OB_UNLIKELY(table_ids.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected empty table id", K(ret), K_(container_tablet_id), K_(schema_version)); + } else if (OB_UNLIKELY(table_ids.at(0) == OB_INVALID_ID)) { + ret = OB_TABLE_IS_DELETED; + LOG_WARN("table is deleted", K(ret), K_(container_tablet_id), K_(schema_version)); + } else { + container_table_id_ = table_ids.at(0); + } + if (FAILEDx(ObMviewCompactionHelper::get_mview_id_from_container_table(container_table_id_, mview_id_))) { + LOG_WARN("Failed to get mview id", K(ret), K(tenant_id), K_(container_table_id)); + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) { + LOG_WARN("Failed to get tenant schema guard", K(ret), K(tenant_id), K_(container_tablet_id)); + } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, container_table_id_, table_schema))) { + LOG_WARN("fail to get table schema", K(ret), K(tenant_id), K_(container_table_id), K_(container_tablet_id)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected null table schema", K(ret), K(tenant_id), K_(container_table_id), K_(container_tablet_id)); + } else { + database_id_ = table_schema->get_database_id(); + } + if (FAILEDx(refresh_scn_range_.start_scn_.convert_for_sql(merge_param.merge_version_range_.base_version_))) { + LOG_WARN("Failed to convert to scn", K(ret), K(merge_param.merge_version_range_)); + } else if (OB_FAIL(refresh_scn_range_.end_scn_.convert_for_sql(merge_param.merge_version_range_.snapshot_version_))) { + LOG_WARN("Failed to convert to scn", K(ret), K(merge_param.merge_version_range_)); + } else if (OB_UNLIKELY(!refresh_scn_range_.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected refresh scn range", K(ret), K_(refresh_scn_range)); + } else if (OB_FAIL(ObMviewCompactionHelper::create_inner_session(merge_param.get_schema()->is_oracle_mode(), database_id_, free_session_ctx, session))) { + LOG_WARN("Failed to create inner session", K(ret),K(tenant_id), K_(container_table_id), K_(container_tablet_id), K(database_id_)); + } else if (OB_FAIL(ObMviewCompactionHelper::generate_mview_refresh_sql(session, schema_guard, table_schema, merge_param.merge_range_, + merge_param.static_param_.rowkey_read_info_, *this))) { + LOG_WARN("Failed to generate mview refresh sql", K(ret), K(tenant_id), K_(container_table_id), K_(container_tablet_id)); + } + if (OB_SUCC(ret)) { + FLOG_INFO("[MVIEW COMPACTION]: success to init mview merge param", K(ret), K(*this)); + } + ObMviewCompactionHelper::release_inner_session(free_session_ctx, session); + return ret; +} + +int64_t ObMviewMergeParameter::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + J_OBJ_START(); + J_KV(K_(database_id), + K_(mview_id), + K_(container_table_id), + K_(container_tablet_id), + K_(schema_version), + K_(refresh_scn_range), + K_(refresh_sql_count)); + J_COMMA(); + if (is_valid()) { + for (int64_t i = 0; i < refresh_sql_count_; ++i) { + J_KV(K(i)); + J_COMMA(); + J_KV("refresh_sql", refresh_sqls_[i]); + J_COMMA(); + } + } + J_OBJ_END(); + return pos; +} + +int ObMviewCompactionHelper::get_mview_id_from_container_table(const uint64_t container_table_id, uint64_t &mview_id) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = MTL_ID(); + ObSqlString sql; + if (OB_FAIL(sql.assign_fmt("SELECT table_id FROM %s WHERE data_table_id = %ld AND table_type = %d", + OB_ALL_TABLE_TNAME, container_table_id, MATERIALIZED_VIEW))) { + LOG_WARN("Failed to assign sql", K(ret)); + } else { + SMART_VAR(ObISQLClient::ReadResult, res) { + sqlclient::ObMySQLResult *result = NULL; + if (OB_FAIL(GCTX.sql_proxy_->read(res, tenant_id, sql.ptr()))) { + LOG_WARN("Failed to execute sql", K(ret), K(sql), K(tenant_id)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected null result", K(ret), K(sql), K(tenant_id)); + } else if (OB_FAIL(result->next())) { + LOG_WARN("Failed to get next result", K(ret), K(sql), K(tenant_id)); + } else { + EXTRACT_INT_FIELD_MYSQL(*result, "table_id", mview_id, uint64_t); + if (OB_FAIL(ret)) { + LOG_WARN("Failed to get int from result", K(ret), K(sql)); + } + } + if (OB_SUCC(ret) && OB_UNLIKELY(OB_ITER_END != result->next())) { + LOG_WARN("Unexpected ret, must be only one result", K(ret)); + ret = OB_ERR_UNEXPECTED; + } + } + } + return ret; +} + +int ObMviewCompactionHelper::generate_mview_refresh_sql( + sql::ObSQLSessionInfo *session, + share::schema::ObSchemaGetterGuard &schema_guard, + const share::schema::ObTableSchema *table_schema, + const blocksstable::ObDatumRange &merge_range, + const ObRowkeyReadInfo *rowkey_read_info, + ObMviewMergeParameter &mview_param) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = MTL_ID(); + int64_t part_idx = OB_INVALID_INDEX; + int64_t sub_part_idx = OB_INVALID_INDEX; + sql::ObMVProvider mv_provider(tenant_id, mview_param.mview_id_); + if (table_schema->is_partitioned_table()) { + if (OB_FAIL(table_schema->get_part_idx_by_tablet(mview_param.container_tablet_id_, part_idx, sub_part_idx))) { + LOG_WARN("Failed to get part idx by tablet", K(ret), K(mview_param)); + } + } + if (OB_SUCC(ret)) { + ObArenaAllocator allocator("MVCompaction", OB_MALLOC_NORMAL_BLOCK_SIZE, tenant_id); + ObNewRange sql_range; + sql_range.table_id_ = mview_param.mview_id_; + if (merge_range.is_whole_range()) { + sql_range.set_whole_range(); + } else if (OB_FAIL(convert_datum_range(allocator, rowkey_read_info, merge_range, sql_range))) { + LOG_WARN("Failed to convert datum range", K(ret), K(mview_param)); + } + if (FAILEDx(mv_provider.init_mv_provider(mview_param.refresh_scn_range_.start_scn_, + mview_param.refresh_scn_range_.end_scn_, + &schema_guard, + session, + part_idx, + sub_part_idx, + sql_range))) { + LOG_WARN("Failed to init mv provider", K(ret), K(tenant_id), K(mview_param)); + } else { + LOG_INFO("[MVIEW COMPACTION] yuanzhe debug", K(part_idx), K(sub_part_idx), K(sql_range), K(merge_range)); + } + } + if (OB_SUCC(ret)) { + ObMviewMergeSQL *refresh_sqls = mview_param.refresh_sqls_; + const common::ObIArray &mv_ops = mv_provider.get_operators(); + if (OB_UNLIKELY(mview_param.refresh_sql_count_ != mv_ops.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected number of refresh sql", K(ret), K(mv_ops.count())); + } + for (int64_t i = 0; OB_SUCC(ret) && i < mv_ops.count(); ++i) { + if (mview_param.is_refresh_sql_v0()) { + refresh_sqls[i].type_ = 0 == i || 2 == i ? ObMviewMergeIterType::MVIEW_DELETE : ObMviewMergeIterType::MVIEW_INSERT; + } else { + refresh_sqls[i].type_ = ObMviewMergeIterType::MVIEW_REPLACE; + } + if (OB_FAIL(refresh_sqls[i].sql_.append(mv_ops.at(i)))) { + LOG_WARN("Failed to append mv sql", K(ret), K(i)); + } + } + } + return ret; +} + +int ObMviewCompactionHelper::create_inner_session( + const bool is_oracle_mode, + const uint64_t database_id, + sql::ObFreeSessionCtx &free_session_ctx, + sql::ObSQLSessionInfo *&session) +{ + int ret = OB_SUCCESS; + session = nullptr; + uint32_t sid = sql::ObSQLSessionInfo::INVALID_SESSID; + uint64_t proxy_sid = 0; + const schema::ObTenantSchema *tenant_info = nullptr; + const ObDatabaseSchema *database_schema = nullptr; + ObSchemaGetterGuard schema_guard; + const uint64_t tenant_id = MTL_ID(); + if (OB_FAIL(GCTX.session_mgr_->create_sessid(sid))) { + LOG_WARN("Failed to create sess id", K(ret), K(tenant_id), K(database_id)); + } else if (OB_FAIL(GCTX.session_mgr_->create_session( + tenant_id, sid, proxy_sid, ObTimeUtility::current_time(), session))) { + GCTX.session_mgr_->mark_sessid_unused(sid); + session = nullptr; + LOG_WARN("Failed to create session", K(ret), K(sid), K(tenant_id), K(database_id)); + } else { + free_session_ctx.sessid_ = sid; + free_session_ctx.proxy_sessid_ = proxy_sid; + } + if (FAILEDx(GCTX.schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) { + LOG_WARN("Failed to get tenant schema guard", K(ret), K(tenant_id), K(database_id)); + } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant_info))) { + LOG_WARN("Failed to get tenant info", K(ret), K(tenant_id), K(database_id)); + } else if (OB_ISNULL(tenant_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected null tenant schema", K(ret), K(tenant_id), K(database_id)); + } else if (OB_FAIL(schema_guard.get_database_schema(tenant_id, database_id, database_schema))) { + LOG_WARN("Failed to get database schema", K(ret), K(tenant_id), K(database_id)); + } else if (OB_ISNULL(database_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected null database schema", K(ret), K(tenant_id), K(database_id)); + } else if (OB_FAIL(session->load_default_sys_variable(false, false))) { + LOG_WARN("Failed to load default sys variable", K(ret), K(tenant_id), K(database_id)); + } else if (OB_FAIL(session->load_default_configs_in_pc())) { + LOG_WARN("Failed to load default configs in pc", K(ret), K(tenant_id), K(database_id)); + } else if (OB_FAIL(session->init_tenant(tenant_info->get_tenant_name(), tenant_id))) { + LOG_WARN("Failed to init tenant in session", K(ret), K(tenant_id), K(database_id)); + } else if (OB_FAIL(session->set_default_database(database_schema->get_database_name()))) { + LOG_WARN("Failed to set default database", K(ret), K(tenant_id), K(database_id)); + } else if (OB_FAIL(set_params_to_session(session))) { + LOG_WARN("Failed to set params to session", K(ret)); + } else { + session->set_inner_session(); + session->set_compatibility_mode(is_oracle_mode ? ObCompatibilityMode::ORACLE_MODE : ObCompatibilityMode::MYSQL_MODE); + session->get_ddl_info().set_major_refreshing_mview(true); + LOG_INFO("[MVIEW COMPACTION]: Succ to create inner session", K(ret), K(tenant_id), K(database_id), KP(session)); + } + if (OB_FAIL(ret)) { + release_inner_session(free_session_ctx, session); + } + return ret; +} + +void ObMviewCompactionHelper::release_inner_session(sql::ObFreeSessionCtx &free_session_ctx, sql::ObSQLSessionInfo *&session) +{ + if (nullptr != session) { + LOG_INFO("[MVIEW COMPACTION]: Release inner session", KP(session)); + session->get_ddl_info().set_major_refreshing_mview(false); + session->set_session_sleep(); + GCTX.session_mgr_->revert_session(session); + GCTX.session_mgr_->free_session(free_session_ctx); + GCTX.session_mgr_->mark_sessid_unused(free_session_ctx.sessid_); + session = nullptr; + } +} + +int ObMviewCompactionHelper::create_inner_connection(sql::ObSQLSessionInfo *session, common::sqlclient::ObISQLConnection *&connection) +{ + int ret = OB_SUCCESS; + observer::ObInnerSQLConnectionPool *conn_pool = static_cast(GCTX.sql_proxy_->get_pool()); + if (OB_FAIL(conn_pool->acquire(session, connection))) { + LOG_WARN("Failed to acquire conn_", K(ret)); + } else { + LOG_INFO("[MVIEW COMPACTION]: Succ to create inner connection", K(ret), KP(session), KP(connection)); + } + return ret; +} + +void ObMviewCompactionHelper::release_inner_connection(common::sqlclient::ObISQLConnection *&connection) +{ + if (nullptr != connection) { + LOG_INFO("[MVIEW COMPACTION]: Release inner connection", KP(connection)); + GCTX.sql_proxy_->get_pool()->release(connection, true); + connection = nullptr; + } +} + +int ObMviewCompactionHelper::set_params_to_session(sql::ObSQLSessionInfo *session) +{ + int ret = OB_SUCCESS; + ObObj param_val; + param_val.set_int(REFRESH_SQL_TIMEOUT_US); + OZ(session->update_sys_variable(SYS_VAR_OB_QUERY_TIMEOUT, param_val)); + if (OB_SUCC(ret)) { + param_val.set_int(ObConsistencyLevel::WEAK); + OZ(session->update_sys_variable(SYS_VAR_OB_READ_CONSISTENCY, param_val)); + } + if (OB_SUCC(ret)) { + ObObj result_val; + if (OB_FAIL(session->get_sys_variable(SYS_VAR_OB_QUERY_TIMEOUT, result_val))) { + LOG_WARN("Failed to get sys var", K(ret)); + } else { + LOG_INFO("[MVIEW COMPACTION]: SYS_VAR_OB_QUERY_TIMEOUT=", K(result_val)); + } + if (FAILEDx(session->get_sys_variable(SYS_VAR_OB_READ_CONSISTENCY, result_val))) { + LOG_WARN("Failed to get sys var", K(ret)); + } else { + LOG_INFO("[MVIEW COMPACTION]: SYS_VAR_OB_READ_CONSISTENCY=", K(result_val)); + } + } + return ret; +} + +int ObMviewCompactionHelper::convert_datum_range( + common::ObIAllocator &allocator, + const storage::ObRowkeyReadInfo *rowkey_read_info, + const blocksstable::ObDatumRange &merge_range, + ObNewRange &sql_range) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(rowkey_read_info)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid null rowkey read info", K(ret)); + } else { + sql_range.border_flag_ = merge_range.get_border_flag(); + const int64_t schema_rowkey_cnt = rowkey_read_info->get_schema_rowkey_count(); + const common::ObIArray &col_descs = rowkey_read_info->get_columns_desc(); + void *buf = nullptr; + ObObj *start_key = nullptr; + ObObj *end_key = nullptr; + if (OB_ISNULL(buf = allocator.alloc(sizeof(ObObj) * schema_rowkey_cnt))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("Failed to alloc memory for start key", K(ret)); + } else if (FALSE_IT(start_key = new (buf) ObObj[schema_rowkey_cnt])) { + } else if (OB_ISNULL(buf = allocator.alloc(sizeof(ObObj) * schema_rowkey_cnt))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("Failed to alloc memory for end key", K(ret)); + } else if (FALSE_IT(end_key = new (buf) ObObj[schema_rowkey_cnt])) { + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < schema_rowkey_cnt; ++i) { + if (i < merge_range.get_start_key().get_datum_cnt()) { + if (OB_FAIL(merge_range.get_start_key().get_datum(i).to_obj_enhance(start_key[i], col_descs.at(i).col_type_))) { + LOG_WARN("Failed to convert to obj", K(ret), K(i)); + } + } else { + start_key[i].set_min_value(); + } + if (OB_FAIL(ret)) { + } else if (i < merge_range.get_end_key().get_datum_cnt()) { + if (OB_FAIL(merge_range.get_end_key().get_datum(i).to_obj_enhance(end_key[i], col_descs.at(i).col_type_))) { + LOG_WARN("Failed to convert to obj", K(ret), K(i)); + } + } else { + end_key[i].set_max_value(); + } + } + if (OB_SUCC(ret)) { + sql_range.start_key_.assign(start_key, schema_rowkey_cnt); + sql_range.end_key_.assign(end_key, schema_rowkey_cnt); + } + } + } + return ret; +} + +} +} diff --git a/src/storage/compaction/ob_mview_compaction_util.h b/src/storage/compaction/ob_mview_compaction_util.h new file mode 100644 index 0000000000..b91357d129 --- /dev/null +++ b/src/storage/compaction/ob_mview_compaction_util.h @@ -0,0 +1,139 @@ +// Copyright (c) 2024 OceanBase +// OceanBase is licensed under Mulan PubL v2. +// You can use this software according to the terms and conditions of the Mulan PubL v2. +// You may obtain a copy of Mulan PubL v2 at: +// http://license.coscl.org.cn/MulanPubL-2.0 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +// EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +// MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PubL v2 for more details. + +#ifndef OCEANBASE_STORAGE_OB_MVIEW_COMPACTION_UTIL_ +#define OCEANBASE_STORAGE_OB_MVIEW_COMPACTION_UTIL_ + +#include "common/ob_tablet_id.h" +#include "common/object/ob_object.h" +#include "lib/string/ob_sql_string.h" +#include "share/ob_table_range.h" + +namespace oceanbase +{ +namespace share +{ +namespace schema +{ +class ObSchemaGetterGuard; +class ObTableSchema; +struct ObColDesc; +} +} + +namespace common +{ +namespace sqlclient +{ +class ObISQLConnection; +} +} +namespace sql +{ +class ObSQLSessionInfo; +class ObFreeSessionCtx; +} +namespace storage +{ +class ObRowkeyReadInfo; +} +namespace blocksstable +{ +struct ObDatumRange; +} +namespace compaction +{ + +struct ObMergeParameter; +class ObPartitionMergeIter; + +enum class ObMviewMergeIterType +{ + MVIEW_INSERT = 0, + MVIEW_DELETE, + MVIEW_REPLACE +}; + +struct ObMviewMergeSQL +{ + OB_INLINE bool is_delete() const + { + return ObMviewMergeIterType::MVIEW_DELETE == type_; + } + OB_INLINE bool is_replace() const + { + return ObMviewMergeIterType::MVIEW_REPLACE == type_; + } + TO_STRING_KV(K_(type), K_(sql)); + ObMviewMergeIterType type_; + ObSqlString sql_; +}; + +struct ObMviewMergeParameter +{ + static const int64_t MAX_REFRESH_SQL_COUNT = 4; + static const int64_t REFRESH_SQL_COUNT_V0 = MAX_REFRESH_SQL_COUNT; + static const int64_t REFRESH_SQL_COUNT_V1 = 2; + ObMviewMergeParameter(); + ~ObMviewMergeParameter(); + int init(const ObMergeParameter &merge_param); + OB_INLINE bool is_valid() const + { + return database_id_ > 0 && mview_id_ > 0 && container_table_id_ > 0 && container_tablet_id_.is_valid() && + schema_version_ > 0 && refresh_scn_range_.is_valid(); + } + OB_INLINE bool is_refresh_sql_v0() const + { + return REFRESH_SQL_COUNT_V0 == refresh_sql_count_; + } + DECLARE_TO_STRING; + uint64_t database_id_; + uint64_t mview_id_; + uint64_t container_table_id_; + ObTabletID container_tablet_id_; + int64_t schema_version_; + share::ObScnRange refresh_scn_range_; // (last_refresh_scn, current_refresh_scn] + int64_t refresh_sql_count_; + ObMviewMergeSQL refresh_sqls_[MAX_REFRESH_SQL_COUNT]; +}; + +class ObMviewCompactionHelper +{ +public: + static const int64_t REFRESH_SQL_TIMEOUT_US = 604800000000; + static int get_mview_id_from_container_table(const uint64_t container_table_id, uint64_t &mview_id); + static int generate_mview_refresh_sql( + sql::ObSQLSessionInfo *session, + share::schema::ObSchemaGetterGuard &schema_guard, + const share::schema::ObTableSchema *table_schema, + const blocksstable::ObDatumRange &merge_range, + const storage::ObRowkeyReadInfo *rowkey_read_info, + ObMviewMergeParameter &mview_param); + static int create_inner_session( + const bool is_oracle_mode, + const uint64_t database_id, + sql::ObFreeSessionCtx &free_session_ctx, + sql::ObSQLSessionInfo *&session); + static void release_inner_session(sql::ObFreeSessionCtx &free_session_ctx, sql::ObSQLSessionInfo *&session); + static int create_inner_connection(sql::ObSQLSessionInfo *session, common::sqlclient::ObISQLConnection *&connection); + static void release_inner_connection(common::sqlclient::ObISQLConnection *&connection); + static int set_params_to_session(sql::ObSQLSessionInfo *session); +private: + static int convert_datum_range( + common::ObIAllocator &allocator, + const storage::ObRowkeyReadInfo *rowkey_read_info, + const blocksstable::ObDatumRange &merge_range, + ObNewRange &sql_range); +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/storage/compaction/ob_partition_merge_iter.cpp b/src/storage/compaction/ob_partition_merge_iter.cpp index aa3fe086db..8724314e81 100644 --- a/src/storage/compaction/ob_partition_merge_iter.cpp +++ b/src/storage/compaction/ob_partition_merge_iter.cpp @@ -17,6 +17,8 @@ #include "storage/access/ob_table_read_info.h" #include "ob_tablet_merge_ctx.h" #include "storage/column_store/ob_column_oriented_sstable.h" +#include "storage/compaction/ob_mview_compaction_util.h" +#include "observer/ob_inner_sql_result.h" namespace oceanbase { @@ -2144,5 +2146,173 @@ int ObPartitionMinorMacroMergeIter::get_curr_macro_block( return ret; } +/* + * ObPartitionRowMergeIter used for mv major merge + */ +ObPartitionMVRowMergeIter::ObPartitionMVRowMergeIter(common::ObIAllocator &allocator) + : ObPartitionMergeIter(allocator), + is_delete_(false), + is_replace_(false), + sql_idx_(-1), + sql_read_col_cnt_(0), + store_col_cnt_(0), + read_result_(), + result_row_(), + free_session_ctx_(), + session_(nullptr), + conn_(nullptr), + sql_result_(nullptr) +{ +} + +ObPartitionMVRowMergeIter::~ObPartitionMVRowMergeIter() +{ + read_result_.~ReadResult(); // need decons before session + ObMviewCompactionHelper::release_inner_connection(conn_); + ObMviewCompactionHelper::release_inner_session(free_session_ctx_, session_); +} + +int ObPartitionMVRowMergeIter::init(const ObMergeParameter &merge_param, + const int64_t refresh_sql_idx, + const ObITableReadInfo *read_info) +{ + int ret = OB_SUCCESS; + int64_t store_column_cnt = 0; + sql_idx_ = refresh_sql_idx; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("ObPartitionMVRowMergeIter init twice", K(ret)); + } else if (OB_UNLIKELY(!merge_param.is_valid() + || refresh_sql_idx < 0 || refresh_sql_idx >= merge_param.mview_merge_param_->refresh_sql_count_ + || nullptr == read_info)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid arguments to init ObPartitionMVRowMergeIter", K(ret), + K(merge_param), K(refresh_sql_idx), KP(read_info)); + } else if (OB_UNLIKELY(!inner_check(merge_param))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid argument to inner init", K(ret), K(*this), K(merge_param)); + } else if (OB_FAIL(inner_init(merge_param))) { + LOG_WARN("Failed to inner init", K(ret)); + } else if (OB_FAIL(merge_param.get_schema()->get_store_column_count(store_column_cnt, true /* full_col, unused */))) { + LOG_WARN("Failed to get store column cnt", K(ret)); + } else if (OB_FAIL(result_row_.init(allocator_, store_column_cnt + storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt()))) { + LOG_WARN("Failed to init result row", K(ret), K(store_column_cnt)); + } else { + is_delete_ = merge_param.mview_merge_param_->refresh_sqls_[sql_idx_].is_delete(); + is_replace_ = merge_param.mview_merge_param_->refresh_sqls_[sql_idx_].is_replace(); + schema_rowkey_column_cnt_ = read_info->get_schema_rowkey_count(); + result_row_.count_ = store_column_cnt + storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); + result_row_.storage_datums_[schema_rowkey_column_cnt_].set_int(-merge_param.merge_version_range_.snapshot_version_); + result_row_.storage_datums_[schema_rowkey_column_cnt_ + 1].set_int(0); + sql_read_col_cnt_ = is_replace_ ? store_column_cnt + 1 : // normal columns + old_new$$ + is_delete_ ? schema_rowkey_column_cnt_ : store_column_cnt; + store_col_cnt_ = is_replace_ ? sql_read_col_cnt_ - 1 : sql_read_col_cnt_; + iter_row_count_ = 0; + iter_row_id_ = -1; + curr_row_ = nullptr; + iter_end_ = false; + is_base_iter_ = false; + is_inited_ = true; + LOG_INFO("[MVIEW COMPACTION]: Succ to init partition mv merge iter", K(ret), K(*this)); + } + return ret; +} + +bool ObPartitionMVRowMergeIter::inner_check(const ObMergeParameter &merge_param) +{ + bool bret = true; + if (OB_UNLIKELY(is_multi_version_merge(merge_param.static_param_.get_merge_type()) || + !merge_param.is_mv_merge())) { + bret = false; + LOG_WARN_RET(OB_ERR_UNEXPECTED, "Unexpected merge type for mv merge", K(merge_param)); + } else if (OB_UNLIKELY(!merge_param.mview_merge_param_->is_valid())) { + bret = false; + LOG_WARN_RET(OB_ERR_UNEXPECTED, "Invalid mv merge param", K(merge_param)); + } + return bret; +} + +int ObPartitionMVRowMergeIter::inner_init(const ObMergeParameter &merge_param) +{ + int ret = OB_SUCCESS; + const ObSqlString &sql = merge_param.mview_merge_param_->refresh_sqls_[sql_idx_].sql_; + if (OB_FAIL(ObMviewCompactionHelper::create_inner_session(merge_param.get_schema()->is_oracle_mode(), + merge_param.mview_merge_param_->database_id_, + free_session_ctx_, session_))) { + LOG_WARN("Failed to create inner session", K(ret), KPC(merge_param.mview_merge_param_)); + } else if (OB_FAIL(ObMviewCompactionHelper::create_inner_connection(session_, conn_))) { + LOG_WARN("Failed to create inner connection", K(ret), K_(sql_idx)); + } else if (OB_FAIL(conn_->execute_read(GCONF.cluster_id, MTL_ID(), sql.ptr(), read_result_))) { + LOG_WARN("Failed to execute", K(ret), K_(sql_idx), K(sql)); + } else if (OB_ISNULL(sql_result_ = static_cast(read_result_.get_result()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected null sql result", K(ret), K_(sql_idx), K(sql)); + } + if (OB_FAIL(ret)) { + ObMviewCompactionHelper::release_inner_connection(conn_); + ObMviewCompactionHelper::release_inner_session(free_session_ctx_, session_); + } + return ret; +} + +int ObPartitionMVRowMergeIter::next() +{ + int ret = OB_SUCCESS; + bool is_delete_row = is_delete_; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObPartitionRowMergeIter is not inited", K(ret), K(*this)); + } else if (OB_UNLIKELY(iter_end_)) { + ret = OB_ITER_END; + } else if (FALSE_IT(curr_row_ = nullptr)) { + } else if (OB_FAIL(sql_result_->next())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("Failed to get next row", K(ret)); + } else { + iter_end_ = true; + } + } else { + const ObNewRow *new_row = sql_result_->get_row(); + if (OB_UNLIKELY(nullptr == new_row || sql_read_col_cnt_ != new_row->get_count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected result row", K(ret), K(schema_rowkey_column_cnt_), K(sql_read_col_cnt_), KPC(new_row)); + } + int64_t read_idx = 0; + for (; OB_SUCC(ret) && read_idx < schema_rowkey_column_cnt_; ++read_idx) { + if (OB_FAIL(result_row_.storage_datums_[read_idx].from_obj(new_row->get_cell(read_idx)))) { + LOG_WARN("Failed to convert obj to datum", K(ret)); + } + } + for (; OB_SUCC(ret) && read_idx < store_col_cnt_; ++read_idx) { + const int64_t datum_idx = read_idx + storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); + if (OB_FAIL(result_row_.storage_datums_[datum_idx].from_obj(new_row->get_cell(read_idx)))) { + LOG_WARN("Failed to convert obj to datum", K(ret)); + } + } + if (OB_SUCC(ret) && is_replace_) { + const ObObj &old_new_obj = new_row->get_cell(read_idx); + if (OB_UNLIKELY(old_new_obj.is_nop_value() || old_new_obj.is_null())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected null old_new$$ value", K(ret), K(old_new_obj)); + } else if (0 == ObString::make_string(ObMviewScanInfo::OLD_ROW).case_compare(old_new_obj.get_string())) { + is_delete_row = true; + } else if (0 == ObString::make_string(ObMviewScanInfo::NEW_ROW).case_compare(old_new_obj.get_string())) { + is_delete_row = false; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected old_new$$ value", K(ret), K(old_new_obj)); + } + } + } + if (OB_SUCC(ret)) { + is_delete_row ? result_row_.row_flag_.set_flag(ObDmlFlag::DF_DELETE) : result_row_.row_flag_.set_flag(ObDmlFlag::DF_INSERT); + curr_row_ = &result_row_; + iter_row_count_++; + iter_row_id_++; + LOG_DEBUG("[MVIEW COMPACTION]: iter next row", K(ret), K(*this), K_(result_row)); + } + return ret; +} + } //compaction } //oceanbase diff --git a/src/storage/compaction/ob_partition_merge_iter.h b/src/storage/compaction/ob_partition_merge_iter.h index 398cf417d1..7e7a4e6028 100644 --- a/src/storage/compaction/ob_partition_merge_iter.h +++ b/src/storage/compaction/ob_partition_merge_iter.h @@ -29,6 +29,7 @@ #include "storage/access/ob_table_access_param.h" #include "storage/access/ob_table_access_context.h" #include "storage/access/ob_micro_block_handle_mgr.h" +#include "sql/session/ob_sql_session_mgr.h" namespace oceanbase { @@ -38,6 +39,10 @@ namespace storage { struct ObTransNodeDMLStat; } +namespace observer +{ +class ObInnerSQLResult; +} namespace compaction { @@ -127,7 +132,7 @@ public: virtual void reset(); virtual int init(const ObMergeParameter &merge_param, const int64_t iter_idx, - const ObITableReadInfo *read_info) override final; + const ObITableReadInfo *read_info) override; virtual int init(const ObMergeParameter &merge_param, ObITable *table, const ObITableReadInfo *read_info) override final; virtual OB_INLINE bool is_iter_end() const override { return iter_end_; } virtual int multi_version_compare(const ObPartitionMergeIter &other, int &cmp_ret) @@ -402,6 +407,34 @@ private: const bool reuse_uncommit_row_; }; +class ObPartitionMVRowMergeIter final : public ObPartitionMergeIter +{ +public: + ObPartitionMVRowMergeIter(common::ObIAllocator &allocator); + virtual ~ObPartitionMVRowMergeIter(); + virtual int init(const ObMergeParameter &merge_param, + const int64_t refresh_sql_idx, + const ObITableReadInfo *read_info) override; + virtual int next() override; + TO_STRING_KV(K_(is_delete), K_(is_replace), K_(sql_idx), K_(sql_read_col_cnt), K_(store_col_cnt), + K_(free_session_ctx), KP_(session), KP_(conn), KP_(sql_result)); +protected: + virtual int inner_init(const ObMergeParameter &merge_param) override; + virtual bool inner_check(const ObMergeParameter &merge_param) override; +private: + bool is_delete_; + bool is_replace_; + int64_t sql_idx_; + int64_t sql_read_col_cnt_; + int64_t store_col_cnt_; + ObISQLClient::ReadResult read_result_; + blocksstable::ObDatumRow result_row_; + sql::ObFreeSessionCtx free_session_ctx_; + sql::ObSQLSessionInfo *session_; + sqlclient::ObISQLConnection *conn_; + observer::ObInnerSQLResult *sql_result_; +}; + static const int64_t DEFAULT_ITER_COUNT = 16; static const int64_t DEFAULT_ITER_ARRAY_SIZE = DEFAULT_ITER_COUNT * sizeof(ObPartitionMergeIter *); typedef common::ObSEArray MERGE_ITER_ARRAY; diff --git a/src/storage/compaction/ob_partition_rows_merger.cpp b/src/storage/compaction/ob_partition_rows_merger.cpp index f33edcda7e..1893163dca 100644 --- a/src/storage/compaction/ob_partition_rows_merger.cpp +++ b/src/storage/compaction/ob_partition_rows_merger.cpp @@ -15,6 +15,7 @@ #include "storage/compaction/ob_partition_rows_merger.h" #include "storage/column_store/ob_column_oriented_sstable.h" #include "storage/compaction/ob_tablet_merge_ctx.h" +#include "storage/compaction/ob_mview_compaction_util.h" namespace oceanbase { @@ -581,7 +582,9 @@ int ObPartitionMergeHelper::init_merge_iters(const ObMergeParameter &merge_param ObSSTable *sstable = nullptr; ObPartitionMergeIter *merge_iter = nullptr; bool is_small_sstable = false; - + if (merge_param.is_mv_merge() && OB_FAIL(init_mv_merge_iters(merge_param))) { + STORAGE_LOG(WARN, "Failed to init mv merge iters", K(ret), K(merge_param)); + } for (int64_t i = table_cnt - 1; OB_SUCC(ret) && i >= 0; i--) { if (OB_ISNULL(table = tables_handle.get_table(i))) { ret = OB_ERR_UNEXPECTED; @@ -623,6 +626,35 @@ int ObPartitionMergeHelper::init_merge_iters(const ObMergeParameter &merge_param return ret; } +int ObPartitionMergeHelper::init_mv_merge_iters(const ObMergeParameter &merge_param) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(0 != merge_iters_.count())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Unexpected iters count in mv merge", K(ret)); + } else { + ObPartitionMergeIter *merge_iter = nullptr; + for (int64_t i = merge_param.mview_merge_param_->refresh_sql_count_ - 1; OB_SUCC(ret) && i >= 0 ; i--) { + if (OB_ISNULL(merge_iter = alloc_helper(allocator_, allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "Failed to alloc memory for mv merge iter", K(ret)); + } else if (OB_FAIL(merge_iter->init(merge_param, i, &read_info_))) { + STORAGE_LOG(WARN, "Failed to init mv merge iter", K(ret), K(i)); + } else if (OB_FAIL(merge_iters_.push_back(merge_iter))) { + STORAGE_LOG(WARN, "Failed to push back mv merge iter", K(ret), KPC(merge_iter)); + } else { + STORAGE_LOG(INFO, "Succ to init mv merge iter", K(ret), K(i), KPC(merge_iter)); + merge_iter = nullptr; + } + if (OB_FAIL(ret) && nullptr != merge_iter) { + merge_iter->~ObPartitionMergeIter(); + allocator_.free(merge_iter); + } + } + } + return ret; +} + int ObPartitionMergeHelper::prepare_rows_merger() { int ret = OB_SUCCESS; @@ -896,6 +928,8 @@ void ObPartitionMergeHelper::reset() if (OB_NOT_NULL(table = iter->get_table())) { FLOG_INFO("partition merge iter row count", K(i), "row_count", iter->get_iter_row_count(), "ghost_row_count", iter->get_ghost_row_count(), "table_key", table->get_key()); + } else { + FLOG_INFO("partition merge iter row count", K(i), "row_count", iter->get_iter_row_count(), KPC(iter)); } iter->~ObPartitionMergeIter(); iter = nullptr; diff --git a/src/storage/compaction/ob_partition_rows_merger.h b/src/storage/compaction/ob_partition_rows_merger.h index cfc1d690cd..e5f0e1c918 100644 --- a/src/storage/compaction/ob_partition_rows_merger.h +++ b/src/storage/compaction/ob_partition_rows_merger.h @@ -187,6 +187,7 @@ protected: virtual ObPartitionMergeIter *alloc_merge_iter(const ObMergeParameter &merge_param, const bool is_base_iter, const bool is_small_sstable, const ObITable *table) = 0; private: int init_merge_iters(const ObMergeParameter &merge_param); + int init_mv_merge_iters(const ObMergeParameter &merge_param); int prepare_rows_merger(); int build_rows_merger(); diff --git a/src/storage/compaction/ob_progressive_merge_helper.cpp b/src/storage/compaction/ob_progressive_merge_helper.cpp index d3fd5e6007..7b4a33e722 100644 --- a/src/storage/compaction/ob_progressive_merge_helper.cpp +++ b/src/storage/compaction/ob_progressive_merge_helper.cpp @@ -153,6 +153,8 @@ int ObProgressiveMergeHelper::init( } else if (FALSE_IT(reset())) { } else if (static_param.is_full_merge_) { full_merge_ = check_macro_need_merge_ = true; + } else if (merge_param.is_mv_merge()) { + STORAGE_LOG(INFO, "mv merge, not init progressive merge", K(ret)); } else { mgr_ = mgr; // init mgr first int64_t rewrite_macro_cnt = 0, reduce_macro_cnt = 0, rewrite_block_cnt_for_progressive = 0; diff --git a/src/storage/compaction/ob_tablet_merge_task.cpp b/src/storage/compaction/ob_tablet_merge_task.cpp index cb151aef86..e28b2ba6b3 100644 --- a/src/storage/compaction/ob_tablet_merge_task.cpp +++ b/src/storage/compaction/ob_tablet_merge_task.cpp @@ -41,6 +41,7 @@ #include "storage/compaction/ob_basic_tablet_merge_ctx.h" #include "storage/compaction/ob_tenant_compaction_progress.h" #include "storage/checkpoint/ob_checkpoint_diagnose.h" +#include "storage/compaction/ob_mview_compaction_util.h" namespace oceanbase { @@ -63,6 +64,7 @@ ObMergeParameter::ObMergeParameter( cg_rowkey_read_info_(nullptr), trans_state_mgr_(nullptr), error_location_(nullptr), + mview_merge_param_(nullptr), allocator_(nullptr) { } @@ -83,6 +85,11 @@ void ObMergeParameter::reset() allocator_->free(cg_rowkey_read_info_); cg_rowkey_read_info_ = nullptr; } + if (nullptr != mview_merge_param_) { + mview_merge_param_->~ObMviewMergeParameter(); + allocator_->free(mview_merge_param_); + mview_merge_param_ = nullptr; + } allocator_ = nullptr; } @@ -136,6 +143,15 @@ int ObMergeParameter::init( } } } + if (OB_SUCC(ret)) { + const ObStorageSchema *schema = nullptr; + if (OB_ISNULL(schema = get_schema())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("schema is null", K(ret), K(*this));; + } else if (schema->is_mv_major_refresh_table() && OB_FAIL(init_mview_merge_param(allocator))) { + STORAGE_LOG(WARN, "failed to init mview merge param", K(ret)); + } + } if (OB_SUCC(ret)) { FLOG_INFO("success to init ObMergeParameter", K(ret), K(idx), K_(static_param_.merge_scn), K_(merge_version_range), K_(merge_rowid_range)); } @@ -164,6 +180,18 @@ int ObMergeParameter::set_merge_rowid_range(ObIAllocator *allocator) return ret; } +int ObMergeParameter::init_mview_merge_param(ObIAllocator *allocator) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(mview_merge_param_ = OB_NEWx(ObMviewMergeParameter, allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "failed to alloc mview merge param", K(ret)); + } else if (OB_FAIL(mview_merge_param_->init(*this))) { + STORAGE_LOG(WARN, "Failed to init mview merge param", K(ret)); + } + return ret; +} + const storage::ObTablesHandleArray & ObMergeParameter::get_tables_handle() const { return static_param_.tables_handle_; diff --git a/src/storage/compaction/ob_tablet_merge_task.h b/src/storage/compaction/ob_tablet_merge_task.h index d8d09361cc..07378e2e0e 100644 --- a/src/storage/compaction/ob_tablet_merge_task.h +++ b/src/storage/compaction/ob_tablet_merge_task.h @@ -42,6 +42,7 @@ struct ObStaticMergeParam; class ObPartitionMerger; struct ObCachedTransStateMgr; class ObPartitionMergeProgress; +class ObMviewMergeParameter; /* DAG : *PrepareTask -> ObTabletMergeTask* -> ObTabletMergeFinishTask @@ -61,6 +62,10 @@ struct ObMergeParameter { const storage::ObTablesHandleArray & get_tables_handle() const; const ObStorageSchema *get_schema() const; bool is_full_merge() const; + OB_INLINE bool is_mv_merge() const + { + return nullptr != mview_merge_param_; + } const ObStaticMergeParam &static_param_; /* rest variables are different for MergeTask */ @@ -70,11 +75,13 @@ struct ObMergeParameter { ObITableReadInfo *cg_rowkey_read_info_; compaction::ObCachedTransStateMgr *trans_state_mgr_; share::ObDiagnoseLocation *error_location_; + ObMviewMergeParameter *mview_merge_param_; ObIAllocator *allocator_; int64_t to_string(char* buf, const int64_t buf_len) const; private: int set_merge_rowid_range(ObIAllocator *allocator); + int init_mview_merge_param(ObIAllocator *allocator); DISALLOW_COPY_AND_ASSIGN(ObMergeParameter); }; diff --git a/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp b/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp index fcf3682998..99dfaccb3a 100644 --- a/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp +++ b/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp @@ -460,14 +460,22 @@ int ObTenantFreezeInfoMgr::get_min_reserved_snapshot( snapshot_info.update_by_smaller_snapshot(ObStorageSnapshotInfo::SNAPSHOT_FOR_UNDO_RETENTION, snapshot_for_undo_retention); snapshot_info.update_by_smaller_snapshot(ObStorageSnapshotInfo::SNAPSHOT_FOR_TX, snapshot_for_tx); snapshot_info.update_by_smaller_snapshot(ObStorageSnapshotInfo::SNAPSHOT_FOR_MAJOR_FREEZE_TS, freeze_info.frozen_scn_.get_val_for_tx()); - - for (int64_t i = 0; i < snapshots.count() && OB_SUCC(ret); ++i) { + bool exist_mview_snapshot_type = false; + for (int64_t i = 0; i < snapshots.count() && OB_SUCC(ret) && !exist_mview_snapshot_type; ++i) { bool related = false; const ObSnapshotInfo &snapshot = snapshots.at(i); if (OB_FAIL(is_snapshot_related_to_tablet(tablet_id, snapshot, related))) { STORAGE_LOG(WARN, "fail to check snapshot relation", K(ret), K(tablet_id), K(snapshot)); } else if (related) { snapshot_info.update_by_smaller_snapshot(snapshot.snapshot_type_, snapshot.snapshot_scn_.get_val_for_tx()); + if (ObSnapShotType::SNAPSHOT_FOR_MAJOR_REFRESH_MV == snapshot.snapshot_type_) { + // if exist mview snapshot type and tenant in restore + if (MTL_TENANT_ROLE_CACHE_IS_RESTORE()) { + exist_mview_snapshot_type = true; + snapshot_info.update_by_smaller_snapshot(ObSnapShotType::SNAPSHOT_FOR_MAJOR_REFRESH_MV, static_cast(0)); + LOG_INFO("exist new mv in restore", K(ret), K(snapshot_info), K(tablet_id), K(merged_version)); + } + } } } LOG_TRACE("check_freeze_info_mgr", K(ret), K(snapshot_info), K(duration), K(snapshot_for_undo_retention), @@ -550,16 +558,19 @@ int ObTenantFreezeInfoMgr::ReloadTask::refresh_merge_info() LOG_INFO("schedule zone to stop major merge", K(tenant_id), K(zone_merge_info), K(global_merge_info)); } else { if (check_tenant_status_) { - bool is_restore = false; - if (OB_FAIL(ObMultiVersionSchemaService::get_instance().check_tenant_is_restore(nullptr, tenant_id, is_restore))) { - LOG_WARN("failed to check tenant is restore", K(ret)); - } else if (is_restore) { - if (REACH_TENANT_TIME_INTERVAL(10L * 1000L * 1000L)) { - LOG_INFO("skip restoring tenant to schedule major merge", K(tenant_id), K(is_restore)); - } - } else { + if (is_sys_tenant(tenant_id) || is_meta_tenant(tenant_id)) { check_tenant_status_ = false; - LOG_INFO("finish check tenant restore", K(tenant_id), K(is_restore)); + } else if (is_virtual_tenant_id(tenant_id)) { // skip virtual tenant + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant is unexpected virtual tenant", KR(ret), K(tenant_id)); + } else { + const ObTenantRole::Role &role = MTL_GET_TENANT_ROLE_CACHE(); + if (is_primary_tenant(role) || is_standby_tenant(role)) { + check_tenant_status_ = false; + LOG_INFO("finish check tenant restore", K(tenant_id), K(role)); + } else if (REACH_TENANT_TIME_INTERVAL(10L * 1000L * 1000L)) { + LOG_INFO("skip restoring tenant to schedule major merge", K(tenant_id), K(role)); + } } } if (!check_tenant_status_) { diff --git a/src/storage/high_availability/ob_ls_migration.cpp b/src/storage/high_availability/ob_ls_migration.cpp index a1009e0995..084158f37f 100644 --- a/src/storage/high_availability/ob_ls_migration.cpp +++ b/src/storage/high_availability/ob_ls_migration.cpp @@ -148,6 +148,8 @@ void ObMigrationCtx::reuse() ObCopyTabletCtx::ObCopyTabletCtx() : tablet_id_(), tablet_handle_(), + macro_block_reuse_mgr_(), + extra_info_(), lock_(common::ObLatchIds::MIGRATE_LOCK), status_(ObCopyTabletStatus::MAX_STATUS) { @@ -170,6 +172,7 @@ void ObCopyTabletCtx::reset() tablet_id_.reset(); tablet_handle_.reset(); status_ = ObCopyTabletStatus::MAX_STATUS; + extra_info_.reset(); } int ObCopyTabletCtx::set_copy_tablet_status(const ObCopyTabletStatus::STATUS &status) @@ -199,6 +202,18 @@ int ObCopyTabletCtx::get_copy_tablet_status(ObCopyTabletStatus::STATUS &status) return ret; } +int ObCopyTabletCtx::get_copy_tablet_record_extra_info(const ObCopyTabletRecordExtraInfo *&extra_info) const +{ + int ret = OB_SUCCESS; + if (!is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("copy tablet ctx is invalid", K(ret), KPC(this)); + } else { + extra_info = &extra_info_; + } + return ret; +} + /******************ObMigrationDagNet*********************/ ObMigrationDagNetInitParam::ObMigrationDagNetInitParam() : arg_(), @@ -1089,6 +1104,12 @@ int ObStartMigrationTask::deal_with_local_ls_() LOG_WARN("leader cannot as add, migrate, change dst", K(ret), K(is_leader), "myaddr", MYADDR, "arg", ctx_->arg_); } +#ifdef ERRSIM + } else if (FALSE_IT(SERVER_EVENT_SYNC_ADD("storage_ha", "before_migration_ls_offline", + "tenant_id", ctx_->tenant_id_, "ls_id", ctx_->arg_.ls_id_.id()))) { + + } else if (FALSE_IT(DEBUG_SYNC(BEFORE_MIGRATION_LS_OFFLINE))) { +#endif } else if (OB_FAIL(ls->offline())) { LOG_WARN("failed to disable log", K(ret), KPC(ctx_)); } else if (ObMigrationOpType::REBUILD_LS_OP == ctx_->arg_.type_) { @@ -2505,7 +2526,8 @@ int ObTabletMigrationDag::inner_reset_status_for_retry() "tenant_id", ctx->tenant_id_, "ls_id", ctx->arg_.ls_id_.id(), "tablet_id", copy_tablet_ctx_.tablet_id_, - "result", result, "retry_count", retry_count); + "result", result, + "retry_count", retry_count); if (OB_FAIL(ctx->ha_table_info_mgr_.remove_tablet_table_info(copy_tablet_ctx_.tablet_id_))) { if (OB_HASH_NOT_EXIST == ret) { ret = OB_SUCCESS; @@ -2514,8 +2536,12 @@ int ObTabletMigrationDag::inner_reset_status_for_retry() } } + DEBUG_SYNC(BEFORE_TABLET_MIGRATION_DAG_INNER_RETRY); + if (OB_SUCC(ret)) { copy_tablet_ctx_.tablet_handle_.reset(); + copy_tablet_ctx_.extra_info_.reset(); + copy_tablet_ctx_.macro_block_reuse_mgr_.reset(); if (OB_ISNULL(ls = ls_handle_.get_ls())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls should not be NULL", K(ret), K(copy_tablet_ctx_)); @@ -2658,6 +2684,8 @@ int ObTabletMigrationTask::process() LOG_WARN("failed to build copy table key info", K(ret), KPC(copy_tablet_ctx_)); } else if (OB_FAIL(build_copy_sstable_info_mgr_())) { LOG_WARN("failed to build copy sstable info mgr", K(ret), KPC(copy_tablet_ctx_)); + } else if (OB_FAIL(ObStorageHAUtils::build_major_sstable_reuse_info(copy_tablet_ctx_->tablet_handle_, copy_tablet_ctx_->macro_block_reuse_mgr_, false /* is_restore */))) { + LOG_WARN("failed to update major sstable reuse info", K(ret), KPC(copy_tablet_ctx_)); } else { #ifdef ERRSIM if (OB_SUCC(ret)) { @@ -2864,6 +2892,10 @@ int ObTabletMigrationTask::generate_physical_copy_task_( } else if (FALSE_IT(init_param.need_sort_macro_meta_ = false)) { } else if (FALSE_IT(init_param.need_check_seq_ = true)) { } else if (FALSE_IT(init_param.ls_rebuild_seq_ = ctx_->local_rebuild_seq_)) { + } else if (FALSE_IT(init_param.macro_block_reuse_mgr_ = ObITable::is_major_sstable(copy_table_key.table_type_) ? ©_tablet_ctx_->macro_block_reuse_mgr_ : nullptr)) { + } else if (FALSE_IT(init_param.extra_info_ = ©_tablet_ctx_->extra_info_)) { + } else if (OB_FAIL(ctx_->ha_table_info_mgr_.get_table_info(copy_tablet_ctx_->tablet_id_, copy_table_key, init_param.sstable_param_))) { + LOG_WARN("failed to get table info", K(ret), KPC(copy_tablet_ctx_), K(copy_table_key)); } else if (!need_copy && FALSE_IT(init_param.sstable_macro_range_info_.copy_table_key_ = copy_table_key)) { } else if (need_copy && OB_FAIL(copy_sstable_info_mgr_.get_copy_sstable_maro_range_info(copy_table_key, init_param.sstable_macro_range_info_))) { LOG_WARN("failed to get copy sstable macro range info", K(ret), K(copy_table_key)); @@ -2917,6 +2949,10 @@ int ObTabletMigrationTask::build_copy_table_key_info_() LOG_WARN("tablet migration task do not init", K(ret)); } else if (OB_FAIL(ctx_->ha_table_info_mgr_.get_table_keys(copy_tablet_ctx_->tablet_id_, copy_table_key_array_))) { LOG_WARN("failed to get copy table keys", K(ret), KPC(copy_tablet_ctx_)); + // sort copy table key array by snapshot version, copy major sstable from low version to high version + } else if (FALSE_IT(ObStorageHAUtils::sort_table_key_array_by_snapshot_version(copy_table_key_array_))) { + } else { + LOG_INFO("succeed to build copy table key info", K(copy_table_key_array_)); } return ret; } diff --git a/src/storage/high_availability/ob_ls_migration.h b/src/storage/high_availability/ob_ls_migration.h index 997846bf00..194306cf4a 100644 --- a/src/storage/high_availability/ob_ls_migration.h +++ b/src/storage/high_availability/ob_ls_migration.h @@ -89,11 +89,14 @@ public: void reset(); int set_copy_tablet_status(const ObCopyTabletStatus::STATUS &status) override; int get_copy_tablet_status(ObCopyTabletStatus::STATUS &status) const override; + int get_copy_tablet_record_extra_info(const ObCopyTabletRecordExtraInfo *&extra_info) const override; VIRTUAL_TO_STRING_KV(K_(tablet_id), K_(status)); public: common::ObTabletID tablet_id_; ObTabletHandle tablet_handle_; + ObMacroBlockReuseMgr macro_block_reuse_mgr_; + ObCopyTabletRecordExtraInfo extra_info_; // extra info of server event private: common::SpinRWLock lock_; ObCopyTabletStatus::STATUS status_; @@ -388,7 +391,6 @@ private: bool &need_copy); int check_transfer_seq_equal_( const ObMigrationTabletParam *src_tablet_meta); - int generate_mds_copy_tasks_( ObTabletCopyFinishTask *tablet_copy_finish_task, share::ObITask *&parent_task); @@ -396,7 +398,6 @@ private: const common::ObIArray ©_table_key_array, common::ObIArray &filter_table_key_array); - private: bool is_inited_; ObMigrationCtx *ctx_; diff --git a/src/storage/high_availability/ob_physical_copy_ctx.cpp b/src/storage/high_availability/ob_physical_copy_ctx.cpp index cb50a46634..577b85b912 100644 --- a/src/storage/high_availability/ob_physical_copy_ctx.cpp +++ b/src/storage/high_availability/ob_physical_copy_ctx.cpp @@ -38,7 +38,11 @@ ObPhysicalCopyCtx::ObPhysicalCopyCtx() need_sort_macro_meta_(true), need_check_seq_(false), ls_rebuild_seq_(-1), - table_key_() + table_key_(), + macro_block_reuse_mgr_(nullptr), + total_macro_count_(0), + reuse_macro_count_(0), + extra_info_(nullptr) { } @@ -57,13 +61,14 @@ bool ObPhysicalCopyCtx::is_valid() const && OB_NOT_NULL(ha_dag_) && OB_NOT_NULL(sstable_index_builder_) && ((need_check_seq_ && ls_rebuild_seq_ >= 0) || !need_check_seq_) - && table_key_.is_valid(); - + && table_key_.is_valid() + && total_macro_count_ >= 0 + && reuse_macro_count_ >= 0 + && OB_NOT_NULL(extra_info_); if (bool_ret) { if (!is_leader_restore_) { bool_ret = src_info_.is_valid(); - } else if (OB_ISNULL(restore_base_info_) - || OB_ISNULL(second_meta_index_store_)) { + } else if (OB_ISNULL(restore_base_info_) || OB_ISNULL(second_meta_index_store_)) { bool_ret = false; } else if (!ObTabletRestoreAction::is_restore_remote_sstable(restore_action_) && !ObTabletRestoreAction::is_restore_replace_remote_sstable(restore_action_) @@ -72,7 +77,6 @@ bool ObPhysicalCopyCtx::is_valid() const LOG_WARN_RET(OB_INVALID_ARGUMENT, "restore_macro_block_id_mgr_ is null", K_(restore_action), KP_(restore_macro_block_id_mgr)); } } - return bool_ret; } @@ -96,6 +100,10 @@ void ObPhysicalCopyCtx::reset() need_check_seq_ = false; ls_rebuild_seq_ = -1; table_key_.reset(); + macro_block_reuse_mgr_ = nullptr; + total_macro_count_ = 0; + reuse_macro_count_ = 0; + extra_info_ = nullptr; } } diff --git a/src/storage/high_availability/ob_physical_copy_ctx.h b/src/storage/high_availability/ob_physical_copy_ctx.h index 5e6b069587..bf05df9e23 100644 --- a/src/storage/high_availability/ob_physical_copy_ctx.h +++ b/src/storage/high_availability/ob_physical_copy_ctx.h @@ -39,8 +39,51 @@ struct ObICopyTabletCtx public: virtual int set_copy_tablet_status(const ObCopyTabletStatus::STATUS &status) = 0; virtual int get_copy_tablet_status(ObCopyTabletStatus::STATUS &status) const = 0; + virtual int get_copy_tablet_record_extra_info(const ObCopyTabletRecordExtraInfo *&extra_info) const = 0; }; +class ObCopyTabletRecordExtraInfo final +{ +public: + ObCopyTabletRecordExtraInfo(); + ~ObCopyTabletRecordExtraInfo(); + void reset(); +public: + OB_INLINE void add_cost_time_ms(const int64_t &time_cost_ms) { ATOMIC_FAA(&cost_time_ms_, time_cost_ms); } + OB_INLINE void add_total_data_size(const int64_t &total_data_size) { ATOMIC_FAA(&total_data_size_, total_data_size); } + OB_INLINE void add_write_data_size(const int64_t &write_data_size) { ATOMIC_FAA(&write_data_size_, write_data_size); } + OB_INLINE void inc_major_count() { ATOMIC_INC(&major_count_); } + OB_INLINE void add_macro_count(const int64_t ¯o_count) { ATOMIC_FAA(¯o_count_, macro_count); } + OB_INLINE void add_major_macro_count(const int64_t &major_macro_count) { ATOMIC_FAA(&major_macro_count_, major_macro_count); } + OB_INLINE void add_reuse_macro_count(const int64_t &reuse_macro_count) { ATOMIC_FAA(&reuse_macro_count_, reuse_macro_count); } + OB_INLINE int get_major_count() const { return ATOMIC_LOAD(&major_count_); } + // not atomic, but only called when major sstable copy finish, which is sequential + int update_max_reuse_mgr_size( + ObMacroBlockReuseMgr *&reuse_mgr); + + TO_STRING_KV(K_(cost_time_ms), K_(total_data_size), K_(write_data_size), K_(major_count), K_(macro_count), + K_(major_macro_count), K_(reuse_macro_count), K_(max_reuse_mgr_size)); +private: + // The following 3 member variables are updated when writer of physical copy task finish + // time cost of tablet copy (fully migration / restore of a single tablet) + int64_t cost_time_ms_; + // total data size reading from copy source (Byte) + int64_t total_data_size_; + // data size writing to dst (Byte) (only count the data size of new macro block) + int64_t write_data_size_; + + // The following 5 member variables are updated when sstable copy finish + // number of major sstable + int64_t major_count_; + // number of all macro block + int64_t macro_count_; + // number of major macro block + int64_t major_macro_count_; + // number of the macro block that is reused + int64_t reuse_macro_count_; + // max reuse mgr size + int64_t max_reuse_mgr_size_; +}; struct ObPhysicalCopyCtx final { @@ -66,7 +109,11 @@ struct ObPhysicalCopyCtx final K_(need_sort_macro_meta), K_(need_check_seq), K_(ls_rebuild_seq), - K_(table_key)); + K_(table_key), + KP_(macro_block_reuse_mgr), + K_(total_macro_count), + K_(reuse_macro_count), + KPC_(extra_info)); common::SpinRWLock lock_; @@ -88,6 +135,10 @@ struct ObPhysicalCopyCtx final bool need_check_seq_; int64_t ls_rebuild_seq_; ObITable::TableKey table_key_; + ObMacroBlockReuseMgr *macro_block_reuse_mgr_; + int total_macro_count_; // total macro block count of single sstable + int reuse_macro_count_; // reuse macro block count of single sstable + ObCopyTabletRecordExtraInfo *extra_info_; private: DISALLOW_COPY_AND_ASSIGN(ObPhysicalCopyCtx); diff --git a/src/storage/high_availability/ob_physical_copy_task.cpp b/src/storage/high_availability/ob_physical_copy_task.cpp index 2490636d0f..fdf4852823 100644 --- a/src/storage/high_availability/ob_physical_copy_task.cpp +++ b/src/storage/high_availability/ob_physical_copy_task.cpp @@ -25,6 +25,7 @@ #include "storage/compaction/ob_refresh_tablet_util.h" #endif #include "storage/tablet/ob_mds_schema_helper.h" +#include "storage/high_availability/ob_storage_ha_utils.h" namespace oceanbase { @@ -32,7 +33,6 @@ using namespace share; using namespace compaction; namespace storage { - /******************ObPhysicalCopyTask*********************/ ObPhysicalCopyTask::ObPhysicalCopyTask() : ObITask(TASK_TYPE_MIGRATE_COPY_PHYSICAL), @@ -94,8 +94,6 @@ int ObPhysicalCopyTask::process() int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; ObMacroBlocksWriteCtx copied_ctx; - int64_t copy_count = 0; - int64_t reuse_count = 0; ObCopyTabletStatus::STATUS status = ObCopyTabletStatus::MAX_STATUS; ObTabletCopyFinishTask *tablet_finish_task = nullptr; @@ -125,6 +123,8 @@ int ObPhysicalCopyTask::process() K(copied_ctx.get_macro_block_count()), K(copied_ctx)); } } + copy_ctx_->total_macro_count_ += copied_ctx.get_macro_block_count(); + copy_ctx_->reuse_macro_count_ += copied_ctx.use_old_macro_block_count_; LOG_INFO("physical copy task finish", K(ret), KPC(copy_macro_range_info_), KPC(copy_ctx_)); } @@ -479,7 +479,7 @@ int ObPhysicalCopyTask::get_macro_block_writer_( ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to alloc memory", K(ret)); } else if (OB_FAIL(tmp_writer->init(copy_ctx_->tenant_id_, copy_ctx_->ls_id_, copy_ctx_->tablet_id_, - this->get_dag()->get_dag_id(), sstable_param, reader, index_block_rebuilder))) { + this->get_dag()->get_dag_id(), sstable_param, reader, index_block_rebuilder, copy_ctx_->extra_info_))) { STORAGE_LOG(WARN, "failed to init macro block writer", K(ret), KPC(copy_ctx_)); } else { writer = tmp_writer; @@ -535,6 +535,7 @@ int ObPhysicalCopyTask::build_copy_macro_block_reader_init_param_( ObCopyMacroBlockReaderInitParam &init_param) { int ret = OB_SUCCESS; + int64_t snapshot_version = 0; if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("physical copy task do not init", K(ret)); @@ -558,11 +559,30 @@ int ObPhysicalCopyTask::build_copy_macro_block_reader_init_param_( init_param.need_check_seq_ = copy_ctx_->need_check_seq_; init_param.ls_rebuild_seq_ = copy_ctx_->ls_rebuild_seq_; init_param.backfill_tx_scn_ = finish_task_->get_sstable_param()->basic_meta_.filled_tx_scn_; - if (!init_param.is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("copy macro block reader init param is invalid", K(ret), K(init_param)); + init_param.macro_block_reuse_mgr_ = copy_ctx_->macro_block_reuse_mgr_; + init_param.data_version_ = 0; + + if (OB_ISNULL(copy_ctx_->macro_block_reuse_mgr_)) { + // skip reuse + } else if (OB_FAIL(copy_ctx_->macro_block_reuse_mgr_->get_major_snapshot_version(copy_ctx_->table_key_, snapshot_version))) { + if (OB_ENTRY_NOT_EXIST != ret) { + LOG_WARN("failed to get reuse major snapshot version", K(ret), KPC(copy_ctx_)); + } else { + ret = OB_SUCCESS; + LOG_INFO("major snapshot version not exist, maybe copying first major in this tablet, skip reuse, set data_version_ to 0", K(ret), KPC(copy_ctx_)); + } } else { - LOG_INFO("succeed init param", KPC(copy_macro_range_info_), K(init_param)); + init_param.data_version_ = snapshot_version; + LOG_INFO("succeed get and set reuse major max snapshot version", K(snapshot_version), KPC(copy_ctx_), K(init_param)); + } + + if (OB_SUCC(ret)) { + if (!init_param.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("copy macro block reader init param is invalid", K(ret), K(init_param)); + } else { + LOG_INFO("succeed init param", KPC(copy_macro_range_info_), K(init_param)); + } } } return ret; @@ -586,6 +606,7 @@ int ObPhysicalCopyTask::record_server_event_() return ret; } -} -} + +} +} diff --git a/src/storage/high_availability/ob_physical_copy_task.h b/src/storage/high_availability/ob_physical_copy_task.h index cf32fb8685..b714fdb574 100644 --- a/src/storage/high_availability/ob_physical_copy_task.h +++ b/src/storage/high_availability/ob_physical_copy_task.h @@ -15,6 +15,7 @@ #include "lib/thread/ob_work_queue.h" #include "lib/thread/ob_dynamic_thread_pool.h" +#include "lib/atomic/ob_atomic.h" #include "share/ob_common_rpc_proxy.h" // ObCommonRpcProxy #include "share/ob_srv_rpc_proxy.h" // ObPartitionServiceRpcProxy #include "share/scheduler/ob_tenant_dag_scheduler.h" @@ -82,7 +83,6 @@ private: int build_copy_macro_block_reader_init_param_( ObCopyMacroBlockReaderInitParam &init_param); int record_server_event_(); - private: // For rebuilder can not retry, define MAX_RETRY_TIMES as 1. static const int64_t MAX_RETRY_TIMES = 1; @@ -93,7 +93,6 @@ private: ObITable::TableKey copy_table_key_; const ObCopyMacroRangeInfo *copy_macro_range_info_; int64_t task_idx_; - DISALLOW_COPY_AND_ASSIGN(ObPhysicalCopyTask); }; diff --git a/src/storage/high_availability/ob_sstable_copy_finish_task.cpp b/src/storage/high_availability/ob_sstable_copy_finish_task.cpp index 686d7be235..628b55ea30 100644 --- a/src/storage/high_availability/ob_sstable_copy_finish_task.cpp +++ b/src/storage/high_availability/ob_sstable_copy_finish_task.cpp @@ -34,6 +34,7 @@ namespace storage //errsim def ERRSIM_POINT_DEF(PHYSICAL_COPY_TASK_GET_TABLET_FAILED); +ERRSIM_POINT_DEF(SSTABLE_COPY_FINISH_TASK_FAILED); // ObPhysicalCopyTaskInitParam ObPhysicalCopyTaskInitParam::ObPhysicalCopyTaskInitParam() @@ -601,6 +602,8 @@ int ObSSTableCopyFinishTask::init(const ObPhysicalCopyTaskInitParam &init_param) copy_ctx_.need_check_seq_ = init_param.need_check_seq_; copy_ctx_.ls_rebuild_seq_ = init_param.ls_rebuild_seq_; copy_ctx_.table_key_ = init_param.sstable_param_->table_key_; + copy_ctx_.macro_block_reuse_mgr_ = init_param.macro_block_reuse_mgr_; + copy_ctx_.extra_info_ = init_param.extra_info_; macro_range_info_index_ = 0; ls_ = init_param.ls_; sstable_param_ = init_param.sstable_param_; @@ -730,10 +733,28 @@ int ObSSTableCopyFinishTask::process() LOG_WARN("failed to create sstable", K(ret), K(copy_ctx_)); } else if (OB_FAIL(check_sstable_valid_())) { LOG_WARN("failed to check sstable valid", K(ret), K(copy_ctx_)); + } else if (OB_FAIL(update_major_sstable_reuse_info_())) { + LOG_WARN("failed to update major sstable reuse info", K(ret), K(copy_ctx_)); + } else if (OB_FAIL(update_copy_tablet_record_extra_info_())) { + LOG_WARN("failed to update copy tablet record extra info", K(ret), K(copy_ctx_)); } else { LOG_INFO("succeed sstable copy finish", K(copy_ctx_)); } +#ifdef ERRSIM + if (OB_SUCC(ret)) { + if (GCONF.errsim_migration_tablet_id == copy_ctx_.tablet_id_.id() + && ObITable::is_major_sstable(copy_ctx_.table_key_.table_type_) + ) { + // inject error when finish copying 3rd major sstable + if (3 == copy_ctx_.extra_info_->get_major_count()) { + ret = SSTABLE_COPY_FINISH_TASK_FAILED ? : OB_SUCCESS; + LOG_ERROR("fake SSTABLE_COPY_FINISH_TASK_FAILED", K(ret), K(copy_ctx_)); + } + } + } +#endif + if (OB_FAIL(ret)) { if (OB_TABLET_NOT_EXIST == ret) { //overwrite ret @@ -753,6 +774,81 @@ int ObSSTableCopyFinishTask::process() return ret; } + +int ObSSTableCopyFinishTask::update_major_sstable_reuse_info_() +{ + int ret = OB_SUCCESS; + int64_t snapshot_version = 0; + int64_t copy_snapshot_version = 0; + int64_t reuse_info_count = 0; + ObTabletHandle tablet_handle; + ObTableHandleV2 table_handle; + ObSSTable *sstable = nullptr; + + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("sstable copy finish task do not init", K(ret)); + } else if (!ObITable::is_major_sstable(copy_ctx_.table_key_.table_type_)) { + // sstable is not major or is meta major, skip reuse + LOG_INFO("sstable is not major, skip update major sstable reuse info", K(ret), K(copy_ctx_)); + } else if (OB_ISNULL(copy_ctx_.macro_block_reuse_mgr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("macro block reuse mgr should not be NULL when migrate or restore major sstable", K(ret), KP(copy_ctx_.macro_block_reuse_mgr_), K(copy_ctx_)); + } else if (OB_FAIL(ls_->ha_get_tablet(copy_ctx_.tablet_id_, tablet_handle))) { + LOG_WARN("failed to get tablet", K(ret), K(copy_ctx_)); + } else if (OB_FAIL(tablet_copy_finish_task_->get_sstable(sstable_param_->table_key_, table_handle))) { + LOG_WARN("failed to get sstable", K(ret), KPC(sstable_param_)); + } else if (OB_FAIL(table_handle.get_sstable(sstable))) { + LOG_WARN("failed to get sstable", K(ret), KPC(sstable_param_)); + } else if (OB_ISNULL(sstable)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sstable should not be NULL", K(ret), KP(sstable), K(tablet_handle)); + } else { + if (OB_FAIL(copy_ctx_.macro_block_reuse_mgr_->update_single_reuse_map(copy_ctx_.table_key_, tablet_handle, *sstable))) { + LOG_WARN("failed to update single reuse map", K(ret), K(copy_ctx_)); + } else if (OB_FAIL(copy_ctx_.macro_block_reuse_mgr_->count(reuse_info_count))) { + LOG_WARN("failed to count reuse info", K(ret), K(copy_ctx_)); + } else { + LOG_INFO("succeed update major sstable reuse info", K(ret), K(copy_ctx_), K(reuse_info_count)); + } + + // if build or count reuse map failed, reset reuse map + if (OB_FAIL(ret)) { + copy_ctx_.macro_block_reuse_mgr_->reset(); + } + } + + return ret; +} + +int ObSSTableCopyFinishTask::update_copy_tablet_record_extra_info_() +{ + int ret = OB_SUCCESS; + + if (OB_ISNULL(copy_ctx_.extra_info_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("copy ctx extra info is NULL", K(ret), K(copy_ctx_)); + } else { + copy_ctx_.extra_info_->add_macro_count(copy_ctx_.total_macro_count_); + copy_ctx_.extra_info_->add_reuse_macro_count(copy_ctx_.reuse_macro_count_); + + if (!ObITable::is_major_sstable(copy_ctx_.table_key_.table_type_)) { + // skip + } else if (OB_FAIL(copy_ctx_.extra_info_->update_max_reuse_mgr_size(copy_ctx_.macro_block_reuse_mgr_))) { + LOG_WARN("failed to update max reuse mgr size", K(ret), K(copy_ctx_)); + } else { + copy_ctx_.extra_info_->inc_major_count(); + copy_ctx_.extra_info_->add_major_macro_count(copy_ctx_.total_macro_count_); + } + } + + if (OB_SUCC(ret)) { + LOG_DEBUG("succeed update copy tablet record extra info", K(copy_ctx_)); + } + + return ret; +} + int ObSSTableCopyFinishTask::prepare_data_store_desc_( const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, diff --git a/src/storage/high_availability/ob_sstable_copy_finish_task.h b/src/storage/high_availability/ob_sstable_copy_finish_task.h index d11c5f49f9..1bb7625fb1 100644 --- a/src/storage/high_availability/ob_sstable_copy_finish_task.h +++ b/src/storage/high_availability/ob_sstable_copy_finish_task.h @@ -60,7 +60,9 @@ struct ObPhysicalCopyTaskInitParam final KP_(second_meta_index_store), K_(need_sort_macro_meta), K_(need_check_seq), - K_(ls_rebuild_seq)); + K_(ls_rebuild_seq), + KP_(macro_block_reuse_mgr), + KPC_(extra_info)); uint64_t tenant_id_; @@ -79,6 +81,8 @@ struct ObPhysicalCopyTaskInitParam final bool need_sort_macro_meta_; // not use bool need_check_seq_; int64_t ls_rebuild_seq_; + ObMacroBlockReuseMgr *macro_block_reuse_mgr_; + ObCopyTabletRecordExtraInfo *extra_info_; private: DISALLOW_COPY_AND_ASSIGN(ObPhysicalCopyTaskInitParam); @@ -214,12 +218,13 @@ private: }; -class ObSSTableCopyFinishTask final : public share::ObITask +class ObSSTableCopyFinishTask : public share::ObITask { public: ObSSTableCopyFinishTask(); virtual ~ObSSTableCopyFinishTask(); - int init(const ObPhysicalCopyTaskInitParam &init_param); + int init( + const ObPhysicalCopyTaskInitParam &init_param); ObPhysicalCopyCtx *get_copy_ctx() { return ©_ctx_; } const ObMigrationSSTableParam *get_sstable_param() { return sstable_param_; } int get_next_macro_block_copy_info( @@ -260,12 +265,25 @@ private: const ObMigrationSSTableParam *sstable_param, compaction::ObMergeType &merge_type); int create_sstable_(); + int create_empty_sstable_(); + int build_create_empty_sstable_param_( + ObTabletCreateSSTableParam ¶m); + int create_sstable_with_index_builder_(); + int build_create_sstable_param_( + ObTablet *tablet, + const blocksstable::ObSSTableMergeRes &res, + ObTabletCreateSSTableParam ¶m); int build_restore_macro_block_id_mgr_( const ObPhysicalCopyTaskInitParam &init_param); int check_sstable_valid_(); int check_sstable_meta_( const ObMigrationSSTableParam &src_meta, const ObSSTableMeta &write_meta); + int update_major_sstable_reuse_info_(); + int update_copy_tablet_record_extra_info_(); + int create_pure_remote_sstable_(); + int build_create_pure_remote_sstable_param_( + ObTabletCreateSSTableParam ¶m); int alloc_and_init_sstable_creator_(ObCopiedSSTableCreatorImpl *&sstable_creator); void free_sstable_creator_(ObCopiedSSTableCreatorImpl *&sstable_creator); int get_space_optimization_mode_( diff --git a/src/storage/high_availability/ob_storage_ha_macro_block_writer.cpp b/src/storage/high_availability/ob_storage_ha_macro_block_writer.cpp index 5ba7e58580..3d3d70b482 100644 --- a/src/storage/high_availability/ob_storage_ha_macro_block_writer.cpp +++ b/src/storage/high_availability/ob_storage_ha_macro_block_writer.cpp @@ -32,7 +32,8 @@ ObStorageHAMacroBlockWriter::ObStorageHAMacroBlockWriter() sstable_param_(nullptr), reader_(NULL), index_block_rebuilder_(nullptr), - macro_checker_() + macro_checker_(), + extra_info_(nullptr) { } @@ -43,22 +44,25 @@ int ObStorageHAMacroBlockWriter::init( const ObDagId &dag_id, const ObMigrationSSTableParam *sstable_param, ObICopyMacroBlockReader *reader, - ObIndexBlockRebuilder *index_block_rebuilder) + ObIndexBlockRebuilder *index_block_rebuilder, + ObCopyTabletRecordExtraInfo *extra_info) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("writer should not be init twice", K(ret)); } else if (OB_INVALID_TENANT_ID == tenant_id - || !ls_id.is_valid() - || !tablet_id.is_valid() - || dag_id.is_invalid() - || OB_ISNULL(sstable_param) - || OB_ISNULL(reader) - || OB_ISNULL(index_block_rebuilder)) { + || !ls_id.is_valid() + || !tablet_id.is_valid() + || dag_id.is_invalid() + || OB_ISNULL(sstable_param) + || OB_ISNULL(reader) + || OB_ISNULL(index_block_rebuilder) + || OB_ISNULL(extra_info)) + { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(tenant_id), K(tablet_id), KP(sstable_param), - KP(reader), KP(index_block_rebuilder)); + KP(reader), KP(index_block_rebuilder), KP(extra_info)); } else if (OB_FAIL(check_sstable_param_for_init_(sstable_param))) { LOG_WARN("failed to check sstable param", K(ret)); } else { @@ -69,6 +73,7 @@ int ObStorageHAMacroBlockWriter::init( sstable_param_ = sstable_param; reader_ = reader; index_block_rebuilder_ = index_block_rebuilder; + extra_info_ = extra_info; is_inited_ = true; } return ret; @@ -133,10 +138,10 @@ int ObStorageHAMacroBlockWriter::process( int64_t start_time = ObTimeUtility::current_time(); blocksstable::ObBufferReader data(NULL, 0, 0); ObStorageObjectOpt opt; + blocksstable::ObDatumRow macro_meta_row; blocksstable::ObStorageObjectWriteInfo write_info; blocksstable::ObStorageObjectHandle write_handle; - blocksstable::ObDataMacroBlockMeta macro_meta; - blocksstable::ObDatumRow macro_meta_row; + ObICopyMacroBlockReader::CopyMacroBlockReadData read_data; copied_ctx.reset(); int64_t write_count = 0; int64_t reuse_count = 0; @@ -145,8 +150,7 @@ int ObStorageHAMacroBlockWriter::process( int64_t write_size = 0; int64_t macro_meta_row_pos = 0; bool is_cancel = false; - obrpc::ObCopyMacroBlockHeader header; - MacroBlockId macro_block_id; + int64_t data_checksum = 0; int32_t result = OB_SUCCESS; if (OB_UNLIKELY(!is_inited_)) { @@ -157,7 +161,6 @@ int ObStorageHAMacroBlockWriter::process( STORAGE_LOG(WARN, "failed to init macro meta row", K(ret)); } else { while (OB_SUCC(ret)) { - macro_block_id.reset(); if (!GCTX.omt_->has_tenant(tenant_id_)) { ret = OB_TENANT_NOT_EXIST; LOG_WARN("tenant not exists, stop migrate", K(ret), K(tenant_id_)); @@ -181,7 +184,7 @@ int ObStorageHAMacroBlockWriter::process( } } else if (OB_FAIL(dag_yield())) { STORAGE_LOG(WARN, "fail to yield dag", KR(ret)); - } else if (OB_FAIL(reader_->get_next_macro_block(header, data, macro_block_id))) { + } else if (OB_FAIL(reader_->get_next_macro_block(read_data))) { if (OB_ITER_END != ret) { LOG_WARN("failed to get next macro block", K(ret)); } else { @@ -189,45 +192,45 @@ int ObStorageHAMacroBlockWriter::process( ret = OB_SUCCESS; } break; - } else if (!header.is_valid() - || (!header.is_reuse_macro_block_ && header.data_type_ != obrpc::ObCopyMacroBlockHeader::DataType::MACRO_DATA) - || (header.is_reuse_macro_block_ && header.data_type_ != obrpc::ObCopyMacroBlockHeader::DataType::MACRO_META_ROW)) { - // invalid argument, if not reuse macro block, buffer must contain whole macro block data + } else if (!read_data.is_valid()) { ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(ERROR, "invalid header", K(ret), K(header)); - } else if (header.is_reuse_macro_block_) { - macro_meta_row_pos = 0; - macro_meta_row.reuse(); - macro_meta.reset(); + STORAGE_LOG(WARN, "invalid read data", K(ret), K(read_data)); + } else if (read_data.is_macro_meta()) { + const MacroBlockId ¯o_id = read_data.macro_meta_->get_macro_id(); - if (OB_FAIL(macro_meta_row.deserialize(data.data(), header.occupy_size_, macro_meta_row_pos))) { - STORAGE_LOG(WARN, "failed to deserialize macro meta row", K(ret), K(data), K(header)); - } else if (OB_FAIL(macro_meta.parse_row(macro_meta_row))) { - STORAGE_LOG(WARN, "failed to parse macro meta row", K(ret), K(macro_meta_row)); - } else if (macro_meta.get_macro_id() == ObIndexBlockRowHeader::DEFAULT_IDX_ROW_MACRO_ID) { + if (ObIndexBlockRowHeader::DEFAULT_IDX_ROW_MACRO_ID == macro_id) { ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "macro id from src has been set to default", K(ret), K(macro_meta)); - } else if (OB_FAIL(copied_ctx.add_macro_block_id(macro_meta.get_macro_id()))) { - STORAGE_LOG(WARN, "fail to add macro id", K(ret), K(macro_meta)); - } else if (OB_FAIL(index_block_rebuilder_->append_macro_row(macro_meta))) { - STORAGE_LOG(WARN, "failed to append macro row", K(ret), K(macro_meta)); + STORAGE_LOG(WARN, "invalid macro id (id is default)", K(ret), K(macro_id)); + } else if (OB_FAIL(copied_ctx.add_macro_block_id(macro_id))) { + STORAGE_LOG(WARN, "fail to add macro id", K(ret), K(macro_id)); + } else if (OB_FAIL(index_block_rebuilder_->append_macro_row(*read_data.macro_meta_))) { + STORAGE_LOG(WARN, "failed to append macro row", K(ret), KPC(read_data.macro_meta_)); } else { + copied_ctx.increment_old_block_count(); ++reuse_count; } - } else if (OB_FAIL(check_macro_block_(data))) { - LOG_ERROR("failed to check macro block, fatal error", K(ret), K(write_count), K(data)); - ret = OB_INVALID_DATA;// overwrite ret - } else if (!write_handle.is_empty() && OB_FAIL(write_handle.wait())) { - LOG_WARN("failed to wait write handle", K(ret), K(write_info)); - } else if (OB_FAIL(set_macro_write_info_(macro_block_id, write_info, opt))) { - LOG_WARN("failed to set macro write info", K(ret), K(macro_block_id)); - } else if (OB_FAIL(write_macro_block_(opt, write_info, write_handle, copied_ctx, data))) { - LOG_WARN("failed to write macro block", K(ret), K(opt), K(macro_block_id)); + } else if (read_data.is_macro_data()) { + ObBufferReader data = read_data.macro_data_; + MacroBlockId macro_block_id = read_data.macro_block_id_; + + if (OB_FAIL(check_macro_block_(data))) { + STORAGE_LOG(WARN, "failed to check macro block, fatal error", K(ret), K(write_count), K(data)); + ret = OB_INVALID_DATA;// overwrite ret + } else if (!write_handle.is_empty() && OB_FAIL(write_handle.wait())) { + STORAGE_LOG(WARN, "failed to wait write handle", K(ret), K(write_info)); + } else if (OB_FAIL(set_macro_write_info_(macro_block_id, write_info, opt))) { + LOG_WARN("failed to set macro write info", K(ret), K(macro_block_id)); + } else if (OB_FAIL(write_macro_block_(opt, write_info, write_handle, copied_ctx, data))) { + LOG_WARN("failed to write macro block", K(ret), K(opt), K(macro_block_id)); + } else { + ObTaskController::get().allow_next_syslog(); + ++write_count; + write_size += data.capacity(); + LOG_INFO("success copy macro block", K(write_count)); + } } else { - ObTaskController::get().allow_next_syslog(); - ++write_count; - write_size += data.capacity(); - LOG_INFO("success copy macro block", K(write_count)); + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "invalid read data", K(ret), K(read_data)); } } @@ -241,20 +244,26 @@ int ObStorageHAMacroBlockWriter::process( } } + data_size = reader_->get_data_size(); + int64_t cost_time_ms = (ObTimeUtility::current_time() - start_time) / 1000; - int64_t data_size_KB = reader_->get_data_size() / 1024; + int64_t data_size_KB = data_size / 1024; int64_t write_size_KB = write_size / 1024; - int64_t rspeed_KB = 0; - int64_t wspeed_KB = 0; + int64_t total_speed_KB = 0; + int64_t write_speed_KB = 0; if (cost_time_ms > 0) { - rspeed_KB = data_size_KB * 1000 / cost_time_ms; - wspeed_KB = write_size_KB * 1000 / cost_time_ms; + total_speed_KB = data_size_KB * 1000 / cost_time_ms; + write_speed_KB = write_size_KB * 1000 / cost_time_ms; } - data_size += reader_->get_data_size(); - LOG_INFO("finish copy macro block data", K(ret), + + extra_info_->add_cost_time_ms(cost_time_ms); + extra_info_->add_total_data_size(data_size); + extra_info_->add_write_data_size(write_size); + + STORAGE_LOG(INFO, "finish copy macro block data", K(ret), "macro_count", copied_ctx.get_macro_block_count(), K(write_count), K(reuse_count), - K(cost_time_ms), "read_size_B", reader_->get_data_size(), K(write_size), K(rspeed_KB), K(wspeed_KB)); + K(cost_time_ms), "read_size_B", data_size, K(write_size), K(total_speed_KB), K(write_speed_KB)); } return ret; diff --git a/src/storage/high_availability/ob_storage_ha_macro_block_writer.h b/src/storage/high_availability/ob_storage_ha_macro_block_writer.h index 0b2c4fd5e9..a3356b456b 100644 --- a/src/storage/high_availability/ob_storage_ha_macro_block_writer.h +++ b/src/storage/high_availability/ob_storage_ha_macro_block_writer.h @@ -18,6 +18,7 @@ #include "storage/blocksstable/ob_macro_block_struct.h" #include "storage/blocksstable/ob_macro_block_checker.h" #include "ob_storage_ha_reader.h" +#include "ob_physical_copy_task.h" #include "storage/blocksstable/index_block/ob_index_block_builder.h" namespace oceanbase @@ -40,6 +41,7 @@ public: }; +class ObCopyTabletRecordExtraInfo; class ObStorageHAMacroBlockWriter : public ObIStorageHAMacroBlockWriter { public: @@ -52,7 +54,9 @@ public: const ObDagId &dag_id, const ObMigrationSSTableParam *sstable_param, ObICopyMacroBlockReader *reader, - ObIndexBlockRebuilder *index_block_rebuilder); + ObIndexBlockRebuilder *index_block_rebuilder, + ObCopyTabletRecordExtraInfo *extra_info + ); virtual int process(blocksstable::ObMacroBlocksWriteCtx &copied_ctx, ObIHADagNetCtx &ha_dag_net_ctx) override; @@ -87,6 +91,7 @@ protected: ObICopyMacroBlockReader *reader_; ObIndexBlockRebuilder *index_block_rebuilder_; blocksstable::ObSSTableMacroBlockChecker macro_checker_; + ObCopyTabletRecordExtraInfo *extra_info_; }; class ObStorageHALocalMacroBlockWriter final : public ObStorageHAMacroBlockWriter diff --git a/src/storage/high_availability/ob_storage_ha_reader.cpp b/src/storage/high_availability/ob_storage_ha_reader.cpp index 17d701e446..b7dd600b3a 100644 --- a/src/storage/high_availability/ob_storage_ha_reader.cpp +++ b/src/storage/high_availability/ob_storage_ha_reader.cpp @@ -36,6 +36,89 @@ namespace storage ERRSIM_POINT_DEF(EN_READER_RPC_NOT_SUPPORT); ERRSIM_POINT_DEF(EN_ONLY_COPY_OLD_VERSION_MAJOR_SSTABLE); +/******************CopyMacroBlockReadInfo*********************/ +ObICopyMacroBlockReader::CopyMacroBlockReadData::CopyMacroBlockReadData() + : data_type_(ObCopyMacroBlockHeader::DataType::MAX), + is_reuse_macro_block_(false), + macro_data_(), + macro_meta_(nullptr), + macro_block_id_(), + allocator_("CopyMacroRead") +{ +} + +ObICopyMacroBlockReader::CopyMacroBlockReadData::~CopyMacroBlockReadData() +{ + reset(); +} + +void ObICopyMacroBlockReader::CopyMacroBlockReadData::reset() +{ + data_type_ = ObCopyMacroBlockHeader::DataType::MAX; + is_reuse_macro_block_ = false; + macro_data_ = ObBufferReader(NULL, 0, 0); + macro_meta_ = nullptr; + macro_block_id_.reset(); + allocator_.reset(); +} + +bool ObICopyMacroBlockReader::CopyMacroBlockReadData::is_valid() const +{ + bool valid = false; + + if (ObCopyMacroBlockHeader::DataType::MACRO_META_ROW == data_type_) { + valid = is_reuse_macro_block_ && OB_NOT_NULL(macro_meta_) && macro_meta_->is_valid(); + } else if (ObCopyMacroBlockHeader::DataType::MACRO_DATA == data_type_) { + valid = !is_reuse_macro_block_ && macro_data_.is_valid(); + } + + return valid; +} + +int ObICopyMacroBlockReader::CopyMacroBlockReadData::set_macro_meta( + const blocksstable::ObDataMacroBlockMeta ¯o_meta, + const bool &is_reuse_macro_block) +{ + int ret = OB_SUCCESS; + + if (!macro_meta.is_valid() || !is_reuse_macro_block) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("set macro meta get invalid argument", K(ret), K(macro_meta), K(is_reuse_macro_block)); + } else if (OB_FAIL(macro_meta.deep_copy(macro_meta_, allocator_))) { + LOG_WARN("failed to deep copy macro meta", K(ret), K(macro_meta)); + } else { + data_type_ = ObCopyMacroBlockHeader::DataType::MACRO_META_ROW; + is_reuse_macro_block_ = is_reuse_macro_block; + } + + return ret; +} + +int ObICopyMacroBlockReader::CopyMacroBlockReadData::set_macro_data( + const ObBufferReader &data, + const bool &is_reuse_macro_block) +{ + int ret = OB_SUCCESS; + + if (!data.is_valid() || is_reuse_macro_block) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("set macro data get invalid argument", K(ret), K(data), K(is_reuse_macro_block)); + } else { + macro_data_ = data; + data_type_ = ObCopyMacroBlockHeader::DataType::MACRO_DATA; + is_reuse_macro_block_ = is_reuse_macro_block; + } + + return ret; +} + +void ObICopyMacroBlockReader::CopyMacroBlockReadData::set_macro_block_id(const MacroBlockId ¯o_block_id) +{ + // won't check macro_block_id_ is valid + macro_block_id_ = macro_block_id; +} + + /******************ObCopyMacroBlockReaderInitParam*********************/ ObCopyMacroBlockReaderInitParam::ObCopyMacroBlockReaderInitParam() : tenant_id_(OB_INVALID_ID), @@ -53,7 +136,9 @@ ObCopyMacroBlockReaderInitParam::ObCopyMacroBlockReaderInitParam() restore_macro_block_id_mgr_(nullptr), need_check_seq_(false), ls_rebuild_seq_(-1), - backfill_tx_scn_() + backfill_tx_scn_(), + data_version_(0), + macro_block_reuse_mgr_(nullptr) { } @@ -65,7 +150,7 @@ bool ObCopyMacroBlockReaderInitParam::is_valid() const { bool bool_ret = false; bool_ret = tenant_id_ != OB_INVALID_ID && ls_id_.is_valid() && table_key_.is_valid() && OB_NOT_NULL(copy_macro_range_info_) - && ((need_check_seq_ && ls_rebuild_seq_ >= 0) || !need_check_seq_); + && ((need_check_seq_ && ls_rebuild_seq_ >= 0) || !need_check_seq_) && data_version_ >= 0; if (bool_ret) { if (!is_leader_restore_) { bool_ret = src_info_.is_valid() && OB_NOT_NULL(bandwidth_throttle_) && OB_NOT_NULL(svr_rpc_proxy_) @@ -142,7 +227,9 @@ ObCopyMacroBlockObReader::ObCopyMacroBlockObReader() allocator_(), macro_block_mem_context_(), last_send_time_(0), - data_size_(0) + data_size_(0), + macro_block_reuse_mgr_(nullptr), + table_key_() { ObMemAttr attr(MTL_ID(), "CMBObReader"); allocator_.set_attr(attr); @@ -181,7 +268,7 @@ int ObCopyMacroBlockObReader::init( arg.ls_id_ = param.ls_id_; arg.table_key_ = param.table_key_; arg.backfill_tx_scn_ = param.backfill_tx_scn_; - arg.data_version_ = 0; + arg.data_version_ = param.data_version_; arg.need_check_seq_ = param.need_check_seq_; arg.ls_rebuild_seq_ = param.ls_rebuild_seq_; LOG_INFO("init arg", K(param), K(arg)); @@ -203,6 +290,8 @@ int ObCopyMacroBlockObReader::init( last_send_time_ = ObTimeUtility::current_time(); data_size_ = rpc_buffer_.get_position(); is_inited_ = true; + macro_block_reuse_mgr_ = param.macro_block_reuse_mgr_; + table_key_ = param.table_key_; LOG_INFO("get first package fetch macro block", K(rpc_buffer_)); } } @@ -275,14 +364,12 @@ int ObCopyMacroBlockObReader::fetch_next_buffer() return ret; } -int ObCopyMacroBlockObReader::get_next_macro_block( - obrpc::ObCopyMacroBlockHeader &header, - blocksstable::ObBufferReader &data, - blocksstable::MacroBlockId ¯o_block_id) +int ObCopyMacroBlockObReader::get_next_macro_block(ObICopyMacroBlockReader::CopyMacroBlockReadData &read_data) { int ret = OB_SUCCESS; - UNUSED(macro_block_id); - header.reset(); + ObBufferReader data; + ObCopyMacroBlockHeader header; + read_data.reset(); if (!is_inited_) { ret = OB_NOT_INIT; @@ -330,10 +417,66 @@ int ObCopyMacroBlockObReader::get_next_macro_block( } } } + + // set read info + if (OB_SUCC(ret)) { + if (OB_FAIL(get_read_info_(header, data, read_data))) { + LOG_WARN("failed to get read info", K(ret)); + } + } } return ret; } +int ObCopyMacroBlockObReader::get_read_info_(const ObCopyMacroBlockHeader &header, const ObBufferReader &data, CopyMacroBlockReadData &read_data) +{ + int ret = OB_SUCCESS; + read_data.reset(); + + if (header.data_type_ == ObCopyMacroBlockHeader::DataType::MACRO_DATA) { + if (OB_FAIL(read_data.set_macro_data(data, header.is_reuse_macro_block_))) { + LOG_WARN("failed to set macro data", K(ret), K(data), K(header)); + } + } else if (header.data_type_ == ObCopyMacroBlockHeader::DataType::MACRO_META_ROW) { + ObDatumRow macro_meta_row; + ObDataMacroBlockMeta macro_meta; + MacroBlockId macro_id; + int64_t data_checksum = 0; + int64_t pos = 0; + + if (OB_FAIL(macro_meta_row.init(OB_MAX_ROWKEY_COLUMN_NUMBER + 1))) { + // use max row key cnt + 1 as capacity, because meta row is kv + LOG_WARN("failed to init macro meta row", K(ret)); + } else if (OB_FAIL(macro_meta_row.deserialize(data.data(), header.occupy_size_, pos))) { + LOG_WARN("failed to deserialize macro meta row", K(ret), K(data), K(header), K(pos)); + } else if (OB_FAIL(macro_meta.parse_row(macro_meta_row))) { + LOG_WARN("failed to parse row", K(ret), K(macro_meta_row)); + } else if (macro_meta.get_macro_id().is_backup_id()) { + // macro block is in backup media, it's migration when quick restore, skip modify macro id + } else if (OB_ISNULL(macro_block_reuse_mgr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("macro block reuse mgr is NULL", K(ret), KP(macro_block_reuse_mgr_)); + } else if (macro_meta.get_macro_id() != ObIndexBlockRowHeader::DEFAULT_IDX_ROW_MACRO_ID) { + ret = OB_INVALID_DATA; + LOG_WARN("local macro id of major sstable hasn't been set to default", K(ret), K(macro_meta)); + } else if (OB_FAIL(macro_block_reuse_mgr_->get_macro_block_reuse_info(table_key_, macro_meta.get_logic_id(), macro_id, data_checksum))) { + LOG_WARN("failed to get macro block reuse info", K(ret), K(table_key_), K(macro_meta.get_logic_id())); + } else if (macro_meta.get_meta_val().data_checksum_ != data_checksum) { + ret = OB_CHECKSUM_ERROR; + LOG_WARN("data checksum not match", K(ret), K(macro_meta.get_meta_val().data_checksum_), K(data_checksum)); + } else { + // modify macro id of macro meta to real local macro id + macro_meta.val_.macro_id_ = macro_id; + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(read_data.set_macro_meta(macro_meta, header.is_reuse_macro_block_))) { + LOG_WARN("failed to set macro meta", K(ret), K(macro_meta)); + } + } + } + return ret; +} /******************ObCopyMacroBlockRestoreReader*********************/ ObCopyMacroBlockRestoreReader::ObCopyMacroBlockRestoreReader() @@ -472,18 +615,12 @@ void ObCopyMacroBlockRestoreReader::reset_buffers_() MEMSET(read_buffer_.data(), 0, read_buffer_.capacity()); } -int ObCopyMacroBlockRestoreReader::get_next_macro_block( - obrpc::ObCopyMacroBlockHeader &header, - blocksstable::ObBufferReader &data, - blocksstable::MacroBlockId ¯o_block_id) +int ObCopyMacroBlockRestoreReader::get_next_macro_block(ObICopyMacroBlockReader::CopyMacroBlockReadData &read_data) { int ret = OB_SUCCESS; - int64_t occupy_size = 0; + read_data.reset(); ObLogicMacroBlockId logic_block_id; - header.reset(); - meta_row_buf_.reuse(); - #ifdef ERRSIM // Simulate minor macro data read failed. if (!table_key_.get_tablet_id().is_ls_inner_tablet() @@ -501,25 +638,12 @@ int ObCopyMacroBlockRestoreReader::get_next_macro_block( } else if (ObTabletRestoreAction::is_restore_remote_sstable(restore_action_)) { // If restore remote sstable, just return backup macro meta to rebuild index/meta tree. blocksstable::ObDataMacroBlockMeta macro_meta; - blocksstable::ObDatumRow macro_meta_row; - common::ObArenaAllocator meta_row_allocator; - backup::ObBackupDeviceMacroBlockId backup_macro_id; if (OB_FAIL(sec_meta_iterator_->get_next(macro_meta))) { LOG_WARN("failed to get next macro meta", K(ret), K(macro_block_count_), KPC(copy_macro_range_info_)); - } else if (FALSE_IT(macro_block_id = macro_meta.get_macro_id())) { - } else if (FALSE_IT(logic_block_id = macro_meta.get_logic_id())) { - } else if (OB_FAIL(macro_meta_row.init(macro_meta.get_meta_val().rowkey_count_ + 1))) { - LOG_WARN("failed to init macro meta row", K(ret), K(macro_meta_row)); - } else if (OB_FAIL(macro_meta.build_row(macro_meta_row, meta_row_allocator))) { - LOG_WARN("failed to build macro row", K(ret), K(macro_meta)); - } else if (OB_FAIL(meta_row_buf_.write_serialize(macro_meta_row))) { - LOG_WARN("failed to write serialize macro meta row into meta row buf", K(ret), K(macro_meta_row), K_(meta_row_buf)); - } else if (FALSE_IT(occupy_size = meta_row_buf_.length())) { + } else if (OB_FAIL(read_data.set_macro_meta(macro_meta, true /* is_reuse_macro_block */))) { + LOG_WARN("failed to set read data", K(ret), K(macro_meta)); } else { - data.assign(meta_row_buf_.data(), occupy_size); - header.occupy_size_ = occupy_size; - header.is_reuse_macro_block_ = true; - header.data_type_ = ObCopyMacroBlockHeader::DataType::MACRO_META_ROW; + logic_block_id = macro_meta.get_logic_id(); } } else { // Otherwise, macro data should be read from backup device. @@ -530,12 +654,16 @@ int ObCopyMacroBlockRestoreReader::get_next_macro_block( share::ObBackupStorageInfo storage_info; share::ObRestoreBackupSetBriefInfo backup_set_brief_info; share::ObBackupDest backup_set_dest; + MacroBlockId macro_block_id; ObStorageIdMod mod; mod.storage_used_mod_ = ObStorageUsedMod::STORAGE_USED_RESTORE; int64_t dest_id = 0; reset_buffers_(); + blocksstable::ObBufferReader data; + data_buffer_.set_pos(0); + read_buffer_.set_pos(0); if (OB_FAIL(ObRestoreUtils::get_backup_data_type(table_key_, data_type))) { LOG_WARN("fail to get backup data type", K(ret), K(table_key_)); } else if (OB_FAIL(fetch_macro_block_index_(macro_block_index_, data_type, logic_block_id, macro_index))) { @@ -554,24 +682,24 @@ int ObCopyMacroBlockRestoreReader::get_next_macro_block( } else if (OB_FAIL(backup::ObLSBackupRestoreUtil::read_macro_block_data(backup_path.get_obstr(), restore_base_info_->backup_dest_.get_storage_info(), mod, macro_index, align_size, read_buffer_, data_buffer_))) { LOG_WARN("failed to read macro block data", K(ret), K(table_key_), K(macro_index), KPC(restore_base_info_)); + } else if (FALSE_IT(data_size_ += data_buffer_.length())) { + } else if (FALSE_IT(data.assign(data_buffer_.data(), data_buffer_.length(), data_buffer_.length()))) { + } else if (OB_FAIL(read_data.set_macro_data(data, false /* is_reuse_macro_block */))) { + LOG_WARN("failed to set read info", K(ret), K(data)); } else { - data_size_ += data_buffer_.length(); - data.assign(data_buffer_.data(), data_buffer_.length(), data_buffer_.length()); - header.is_reuse_macro_block_ = false; - header.data_type_ = ObCopyMacroBlockHeader::MACRO_DATA; - header.occupy_size_ = data_buffer_.length(); + read_data.set_macro_block_id(macro_block_id); } } - if (OB_SUCC(ret)) { - macro_block_count_++; - macro_block_index_++; - if (macro_block_count_ == copy_macro_range_info_->macro_block_count_) { - if (logic_block_id != copy_macro_range_info_->end_macro_block_id_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get macro block end macro block id is not equal to macro block range", - K(ret), K_(macro_block_count), K_(macro_block_index), K(logic_block_id), - "end_macro_block_id", copy_macro_range_info_->end_macro_block_id_, K(table_key_)); + if (OB_SUCC(ret)){ + macro_block_count_++; + macro_block_index_++; + if (macro_block_count_ == copy_macro_range_info_->macro_block_count_) { + if (logic_block_id != copy_macro_range_info_->end_macro_block_id_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get macro block end macro block id is not equal to macro block range", + K(ret), K_(macro_block_count), K_(macro_block_index), K(logic_block_id), + "end_macro_block_id", copy_macro_range_info_->end_macro_block_id_, K(table_key_)); } } } @@ -791,14 +919,11 @@ void ObCopyDDLMacroBlockRestoreReader::reset_buffers_() MEMSET(read_buffer_.data(), 0, read_buffer_.capacity()); } -int ObCopyDDLMacroBlockRestoreReader::get_next_macro_block( - obrpc::ObCopyMacroBlockHeader &header, - blocksstable::ObBufferReader &data, - blocksstable::MacroBlockId ¯o_block_id) +int ObCopyDDLMacroBlockRestoreReader::get_next_macro_block(ObICopyMacroBlockReader::CopyMacroBlockReadData &read_data) { int ret = OB_SUCCESS; int64_t occupy_size = 0; - header.reset(); + read_data.reset(); if (OB_FAIL(ret)) { } else if (!is_inited_) { @@ -842,12 +967,11 @@ int ObCopyDDLMacroBlockRestoreReader::get_next_macro_block( } else if (OB_FAIL(backup::ObLSBackupRestoreUtil::read_macro_block_data(backup_path.get_obstr(), restore_base_info_->backup_dest_.get_storage_info(), mod, macro_index, align_size, read_buffer_, data_buffer_))) { LOG_WARN("failed to read macro block data", K(ret), K(table_key_), K(macro_index), K(physic_block_id), KPC(restore_base_info_)); + } else if (OB_FAIL(read_data.set_macro_data(data_buffer_, false /* is_reuse_macro_block */))) { + LOG_WARN("failed to set read info", K(ret), K(data_buffer_)); + } else if (FALSE_IT(read_data.set_macro_block_id(link_item_.at(macro_block_count_).macro_id_))) { } else { data_size_ += data_buffer_.length(); - data.assign(data_buffer_.data(), data_buffer_.length(), data_buffer_.length()); - header.is_reuse_macro_block_ = false; - header.occupy_size_ = data_buffer_.length(); - macro_block_id = link_item_.at(macro_block_count_).macro_id_; macro_block_count_++; macro_block_index_++; } @@ -1051,7 +1175,6 @@ int ObCopyMacroBlockObProducer::get_next_macro_block( } else { if (copy_macro_block_handle_[handle_idx_].is_reuse_macro_block_) { // only copy macro meta when reuse macro block - // TODO(jyx441808): if is reuse and macro id is local, use reuse mgr to modify macro id; if is reuse and macro id is remote, skip use reuse mgr, just return meta blocksstable::ObDatumRow macro_meta_row; common::ObArenaAllocator meta_row_allocator; // use temporary allocator to get datum row int64_t pos = 0; @@ -1079,6 +1202,7 @@ int ObCopyMacroBlockObProducer::get_next_macro_block( } else { blocksstable::ObMacroBlockCommonHeader common_header; int64_t pos = 0; + if (OB_FAIL(common_header.deserialize( copy_macro_block_handle_[handle_idx_].read_handle_.get_buffer(), copy_macro_block_handle_[handle_idx_].read_handle_.get_data_size(), pos))) { @@ -1091,8 +1215,8 @@ int ObCopyMacroBlockObProducer::get_next_macro_block( occupy_size = common_header.get_header_size() + common_header.get_payload_size(); data.assign(copy_macro_block_handle_[handle_idx_].read_handle_.get_buffer(), occupy_size); copy_macro_block_header.is_reuse_macro_block_ = false; - copy_macro_block_header.data_type_ = ObCopyMacroBlockHeader::DataType::MACRO_DATA; copy_macro_block_header.occupy_size_ = occupy_size; + copy_macro_block_header.data_type_ = ObCopyMacroBlockHeader::DataType::MACRO_DATA; } } } @@ -1125,15 +1249,20 @@ int ObCopyMacroBlockObProducer::prefetch_() // no need to LOG_INFO("has finish, no need do prefetch", K(macro_idx_), K(copy_macro_range_info_)); } else { + int64_t copy_snapshot_version = 0; + if (OB_FAIL(second_meta_iterator_.get_next(macro_meta))) { LOG_WARN("failed to get next macro meta", K(ret), K(macro_idx_), K(copy_macro_range_info_)); } else if (OB_FAIL(copy_macro_block_handle_[handle_idx_].set_macro_meta(macro_meta))) { LOG_WARN("failed to set macro meta", K(ret), K(macro_meta)); } else if (macro_meta.get_logic_id().logic_version_ <= data_version_ || macro_meta.get_macro_id().is_backup_id()) { - // TODO(jyx441808): if logic_version_ <= data_version_ and macro_id is local, can reuse - // if macro_id is not local, only pass macro meta copy_macro_block_handle_[handle_idx_].is_reuse_macro_block_ = true; + // if macro block is local, reset macro meta id to default + // DEFAULT_IDX_ROW_MACRO_ID is also local id + if (macro_meta.get_macro_id().is_local_id()) { + copy_macro_block_handle_[handle_idx_].macro_meta_->val_.macro_id_ = ObIndexBlockRowHeader::DEFAULT_IDX_ROW_MACRO_ID; + } } else { copy_macro_block_handle_[handle_idx_].is_reuse_macro_block_ = false; @@ -1154,7 +1283,7 @@ int ObCopyMacroBlockObProducer::prefetch_() if (OB_SUCC(ret)) { LOG_INFO("do prefetch", K(macro_idx_), "macro block count",copy_macro_range_info_.macro_block_count_ , - "logical id", macro_meta.get_logic_id(), "physical id", macro_meta.get_macro_id()); + "logical id", macro_meta.get_logic_id(), "physical id", macro_meta.get_macro_id(), K(data_version_), "src macro version", copy_snapshot_version); } } return ret; @@ -1845,7 +1974,7 @@ int ObCopySSTableInfoRestoreReader::inner_get_backup_sstable_metas_( bool check_valid = true; for (int64_t i = 0; check_valid && i < backup_sstable_meta_array.count(); ++i) { const backup::ObBackupSSTableMeta &sstable_meta = backup_sstable_meta_array.at(i); - if (!sstable_meta.sstable_meta_.table_key_.is_column_store_sstable()) { + if (!sstable_meta.sstable_meta_.table_key_.is_column_store_sstable() && !sstable_meta.is_major_compaction_mview_dep_) { check_valid = false; } } @@ -3975,7 +4104,8 @@ ObCopyRemoteSSTableMacroBlockRestoreReader::ObCopyRemoteSSTableMacroBlockRestore datum_range_(), macro_block_count_(0), data_size_(0), - meta_row_buf_("CopyMacroMetaRow") + meta_row_buf_("CopyMacroMetaRow"), + macro_block_reuse_mgr_(nullptr) { ObMemAttr attr(MTL_ID(), "CMBReReader"); allocator_.set_attr(attr); @@ -4051,6 +4181,8 @@ int ObCopyRemoteSSTableMacroBlockRestoreReader::init( copy_macro_range_info_ = param.copy_macro_range_info_; restore_base_info_ = param.restore_base_info_; second_meta_index_store_ = param.second_meta_index_store_; + data_version_ = param.data_version_; + macro_block_reuse_mgr_ = param.macro_block_reuse_mgr_; is_inited_ = true; LOG_INFO("succeed to init macro block producer", K(param)); } @@ -4059,20 +4191,14 @@ int ObCopyRemoteSSTableMacroBlockRestoreReader::init( return ret; } -int ObCopyRemoteSSTableMacroBlockRestoreReader::get_next_macro_block( - obrpc::ObCopyMacroBlockHeader &header, - blocksstable::ObBufferReader &data, - blocksstable::MacroBlockId ¯o_block_id) +int ObCopyRemoteSSTableMacroBlockRestoreReader::get_next_macro_block(ObICopyMacroBlockReader::CopyMacroBlockReadData &read_data) { int ret = OB_SUCCESS; blocksstable::ObDataMacroBlockMeta macro_meta; - blocksstable::ObDatumRow macro_meta_row; - common::ObArenaAllocator meta_row_allocator; + MacroBlockId macro_block_id; + ObLogicMacroBlockId logic_block_id; int occupy_size = 0; - header.reset(); - meta_row_buf_.reuse(); - if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret)); @@ -4082,6 +4208,7 @@ int ObCopyRemoteSSTableMacroBlockRestoreReader::get_next_macro_block( } else if (OB_FAIL(second_meta_iterator_.get_next(macro_meta))) { LOG_WARN("failed to get next macro meta", K(ret), K(macro_block_count_), KPC(copy_macro_range_info_)); } else if (FALSE_IT(macro_block_id = macro_meta.get_macro_id())) { + } else if (FALSE_IT(read_data.set_macro_block_id(macro_block_id))) { } else if (!macro_meta.get_macro_id().is_backup_id()) { // This is a local macro block, reuse it. if (!macro_meta.get_macro_id().is_local_id()) { @@ -4091,30 +4218,48 @@ int ObCopyRemoteSSTableMacroBlockRestoreReader::get_next_macro_block( } else if (table_key_.is_column_store_sstable() && sstable_->is_small_sstable()) { // TODO:(wangxiaohui.wxh): fix me // avoid reusing macro block in small sstable, as rebuild index will be failed. - if (OB_FAIL(read_local_macro_block_data_(macro_meta, header, data))) { + if (OB_FAIL(read_local_macro_block_data_(macro_meta, read_data))) { LOG_WARN("failed to read local macro block data", K(ret), K(macro_meta)); } - } else if (OB_FAIL(macro_meta_row.init(macro_meta.get_meta_val().rowkey_count_ + 1))) { - LOG_WARN("failed to init macro meta row", K(ret), K(macro_meta_row)); - } else if (OB_FAIL(macro_meta.build_row(macro_meta_row, meta_row_allocator))) { - LOG_WARN("failed to build macro row", K(ret), K(macro_meta)); - } else if (OB_FAIL(meta_row_buf_.write_serialize(macro_meta_row))) { - LOG_WARN("failed to write serialize macro meta row into meta row buf", K(ret), K(macro_meta_row), K_(meta_row_buf)); - } else if (FALSE_IT(occupy_size = meta_row_buf_.length())) { - } else { - data.assign(meta_row_buf_.data(), occupy_size); - header.occupy_size_ = occupy_size; - header.is_reuse_macro_block_ = true; - header.data_type_ = ObCopyMacroBlockHeader::DataType::MACRO_META_ROW; - macro_block_count_++; + } else if (OB_FAIL(read_data.set_macro_meta(macro_meta, true /* is_reuse_macro_block */))){ + LOG_WARN("failed to set macro meta", K(ret), K(macro_meta)); + } + } else if (macro_meta.get_logic_id().logic_version_ <= data_version_) { + int64_t data_checksum = 0; + MacroBlockId macro_id; + + if (OB_ISNULL(macro_block_reuse_mgr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("macro block reuse mgr is NULL", K(ret), K(macro_meta), KP(macro_block_reuse_mgr_)); + } else if (OB_FAIL(macro_block_reuse_mgr_->get_macro_block_reuse_info(table_key_, macro_meta.get_logic_id(), macro_id, data_checksum))) { + LOG_WARN("failed to get macro block", K(ret), K(macro_meta)); + } else if (data_checksum != macro_meta.get_meta_val().data_checksum_) { + ret = OB_INVALID_DATA; + LOG_WARN("data checksum get from reuse mgr not match", K(ret), K(macro_meta), K(data_checksum)); + } else if (FALSE_IT(macro_meta.val_.macro_id_ = macro_id)) { + } else if (OB_FAIL(read_data.set_macro_meta(macro_meta, true /* is_reuse_macro_block */))) { + LOG_WARN("failed to set macro meta", K(ret), K(macro_meta)); } } else { // This is a backup macro block. - if (OB_FAIL(read_backup_macro_block_data_(macro_meta, header, data))) { + if (OB_FAIL(read_backup_macro_block_data_(macro_meta, read_data))) { LOG_WARN("failed to read backup macro block data", K(ret), K(macro_meta)); } } + if (OB_SUCC(ret)) { + macro_block_count_++; + logic_block_id = macro_meta.get_logic_id(); + if (macro_block_count_ == copy_macro_range_info_->macro_block_count_) { + if (logic_block_id != copy_macro_range_info_->end_macro_block_id_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get macro block end macro block id is not equal to macro block range", + K(ret), K_(macro_block_count), K(logic_block_id), + "end_macro_block_id", copy_macro_range_info_->end_macro_block_id_, K(table_key_)); + } + } + } + return ret; } @@ -4153,8 +4298,7 @@ int ObCopyRemoteSSTableMacroBlockRestoreReader::alloc_buffers_() int ObCopyRemoteSSTableMacroBlockRestoreReader::read_local_macro_block_data_( blocksstable::ObDataMacroBlockMeta ¯o_meta, - obrpc::ObCopyMacroBlockHeader &header, - blocksstable::ObBufferReader &data) + ObICopyMacroBlockReader::CopyMacroBlockReadData &read_data) { int ret = OB_SUCCESS; blocksstable::ObStorageObjectReadInfo read_info; @@ -4162,6 +4306,7 @@ int ObCopyRemoteSSTableMacroBlockRestoreReader::read_local_macro_block_data_( blocksstable::ObMacroBlockCommonHeader common_header; int64_t pos = 0; int64_t occupy_size = 0; + read_data.reset(); read_info.macro_block_id_ = macro_meta.get_macro_id(); read_info.offset_ = sstable_->get_macro_offset(); @@ -4183,27 +4328,29 @@ int ObCopyRemoteSSTableMacroBlockRestoreReader::read_local_macro_block_data_( ret = OB_INVALID_DATA; LOG_ERROR("invalid common header", K(ret), K(read_handle), K(pos), K(common_header)); } else { + blocksstable::ObBufferReader data; occupy_size = common_header.get_header_size() + common_header.get_payload_size(); data.assign(read_handle.get_buffer(), occupy_size, occupy_size); - header.is_reuse_macro_block_ = false; - header.data_type_ = ObCopyMacroBlockHeader::DataType::MACRO_DATA; - header.occupy_size_ = occupy_size; - macro_block_count_++; - LOG_INFO("local small sstable macro data", K(read_handle), K(common_header), K(occupy_size)); + + if (OB_FAIL(read_data.set_macro_data(data, false /* is_reuse_macro_block */))) { + LOG_WARN("failed to set read_info macro data", K(ret), K(read_handle), K(common_header), K(occupy_size)); + } else { + LOG_INFO("local small sstable macro data", K(read_handle), K(common_header), K(occupy_size)); + } } return ret; } int ObCopyRemoteSSTableMacroBlockRestoreReader::read_backup_macro_block_data_( blocksstable::ObDataMacroBlockMeta ¯o_meta, - obrpc::ObCopyMacroBlockHeader &header, - blocksstable::ObBufferReader &data) - + ObICopyMacroBlockReader::CopyMacroBlockReadData &read_data) { int ret = OB_SUCCESS; ObRestoreMacroBlockId restore_macro_id; backup::ObBackupDeviceMacroBlockId backup_macro_id; backup::ObBackupMacroBlockIndex macro_index; + blocksstable::ObBufferReader data; + if (OB_FAIL(backup_macro_id.set(macro_meta.get_macro_id()))) { LOG_WARN("failed to set restore macro id", K(ret), K(macro_meta)); } else if (OB_FAIL(restore_macro_id.set(macro_meta.get_logic_id(), backup_macro_id))) { @@ -4213,22 +4360,11 @@ int ObCopyRemoteSSTableMacroBlockRestoreReader::read_backup_macro_block_data_( } else if (FALSE_IT(backup_macro_data_buffer_.set_pos(0))) { } else if (OB_FAIL(do_read_backup_macro_block_data_(macro_index, backup_macro_data_buffer_))) { LOG_WARN("failed to read backup macro block data", K(ret), K(restore_macro_id), K(macro_index)); + } else if (FALSE_IT(data.assign(backup_macro_data_buffer_.data(), backup_macro_data_buffer_.length(), backup_macro_data_buffer_.length()))) { + } else if (OB_FAIL(read_data.set_macro_data(data, false /* is_reuse_macro_block */))) { + LOG_WARN("failed to set read_info macro data", K(ret), K(data), K(backup_macro_data_buffer_)); } else { - data.assign(backup_macro_data_buffer_.data(), backup_macro_data_buffer_.length(), backup_macro_data_buffer_.length()); - header.is_reuse_macro_block_ = false; - header.data_type_ = ObCopyMacroBlockHeader::DataType::MACRO_DATA; - header.occupy_size_ = backup_macro_data_buffer_.length(); - data_size_ += backup_macro_data_buffer_.length(); - macro_block_count_++; - - if (macro_block_count_ == copy_macro_range_info_->macro_block_count_) { - if (restore_macro_id.logic_block_id_ != copy_macro_range_info_->end_macro_block_id_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get macro block end macro block id is not equal to macro block range", - K(ret), K_(macro_block_count), K(restore_macro_id), KPC(copy_macro_range_info_), K(table_key_)); - } - } } return ret; diff --git a/src/storage/high_availability/ob_storage_ha_reader.h b/src/storage/high_availability/ob_storage_ha_reader.h index 99807dfe9a..65029a002f 100644 --- a/src/storage/high_availability/ob_storage_ha_reader.h +++ b/src/storage/high_availability/ob_storage_ha_reader.h @@ -42,13 +42,32 @@ public: DDL_MACRO_BLOCK_RESTORE_READER = 3, MAX_READER_TYPE }; + struct CopyMacroBlockReadData final + { + public: + CopyMacroBlockReadData(); + ~CopyMacroBlockReadData(); + void reset(); + bool is_valid() const; + int set_macro_meta(const blocksstable::ObDataMacroBlockMeta& macro_meta, const bool &is_reuse_macro_block); + int set_macro_data(const blocksstable::ObBufferReader& macro_data, const bool &is_reuse_macro_block); + void set_macro_block_id(const blocksstable::MacroBlockId ¯o_block_id); + bool is_reuse_macro_block() const { return is_reuse_macro_block_; } + bool is_macro_data() const { return data_type_ == ObCopyMacroBlockHeader::DataType::MACRO_DATA; } + bool is_macro_meta() const { return data_type_ == ObCopyMacroBlockHeader::DataType::MACRO_META_ROW; } + public: + TO_STRING_KV(K_(data_type), K_(is_reuse_macro_block), K_(macro_data), KPC_(macro_meta)); + ObCopyMacroBlockHeader::DataType data_type_; + bool is_reuse_macro_block_; + blocksstable::ObBufferReader macro_data_; + blocksstable::ObDataMacroBlockMeta *macro_meta_; + blocksstable::MacroBlockId macro_block_id_; + common::ObArenaAllocator allocator_; + }; // macro block list is set in the init func ObICopyMacroBlockReader() {} virtual ~ObICopyMacroBlockReader() {} - virtual int get_next_macro_block( - obrpc::ObCopyMacroBlockHeader &header, - blocksstable::ObBufferReader &data, - blocksstable::MacroBlockId ¯o_block_id) = 0; + virtual int get_next_macro_block(CopyMacroBlockReadData &read_data) = 0; virtual Type get_type() const = 0; virtual int64_t get_data_size() const = 0; private: @@ -66,7 +85,7 @@ struct ObCopyMacroBlockReaderInitParam final TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(table_key), KPC_(copy_macro_range_info), K_(src_info), K_(is_leader_restore), K_(restore_action), KP_(bandwidth_throttle), KP_(svr_rpc_proxy), KP_(restore_base_info), KP_(meta_index_store), KP_(second_meta_index_store), - KP_(restore_macro_block_id_mgr)); + KP_(restore_macro_block_id_mgr), KP_(macro_block_reuse_mgr)); uint64_t tenant_id_; share::ObLSID ls_id_; @@ -84,7 +103,8 @@ struct ObCopyMacroBlockReaderInitParam final bool need_check_seq_; int64_t ls_rebuild_seq_; share::SCN backfill_tx_scn_; - + int64_t data_version_; // max snapshot version of local major sstable (use for macro block reuse) + ObMacroBlockReuseMgr *macro_block_reuse_mgr_; private: DISALLOW_COPY_AND_ASSIGN(ObCopyMacroBlockReaderInitParam); }; @@ -95,10 +115,7 @@ public: ObCopyMacroBlockObReader(); virtual ~ObCopyMacroBlockObReader(); int init(const ObCopyMacroBlockReaderInitParam ¶m); - virtual int get_next_macro_block( - obrpc::ObCopyMacroBlockHeader &header, - blocksstable::ObBufferReader &data, - blocksstable::MacroBlockId ¯o_block_id); + virtual int get_next_macro_block(ObICopyMacroBlockReader::CopyMacroBlockReadData &read_info); virtual Type get_type() const { return MACRO_BLOCK_OB_READER; } virtual int64_t get_data_size() const { return data_size_; } @@ -107,7 +124,7 @@ private: int fetch_next_buffer_if_need(); int fetch_next_buffer(); int alloc_from_memctx_first(char* &buf); - + int get_read_info_(const ObCopyMacroBlockHeader &header, const ObBufferReader &data, CopyMacroBlockReadData &read_info); private: bool is_inited_; obrpc::ObStorageRpcProxy::SSHandle handle_; @@ -119,6 +136,8 @@ private: common::ObMacroBlockSizeMemoryContext macro_block_mem_context_; int64_t last_send_time_; int64_t data_size_; + ObMacroBlockReuseMgr *macro_block_reuse_mgr_; + ObITable::TableKey table_key_; DISALLOW_COPY_AND_ASSIGN(ObCopyMacroBlockObReader); }; @@ -128,10 +147,7 @@ public: ObCopyMacroBlockRestoreReader(); virtual ~ObCopyMacroBlockRestoreReader(); int init(const ObCopyMacroBlockReaderInitParam ¶m); - virtual int get_next_macro_block( - obrpc::ObCopyMacroBlockHeader &header, - blocksstable::ObBufferReader &data, - blocksstable::MacroBlockId ¯o_block_id); + virtual int get_next_macro_block(ObICopyMacroBlockReader::CopyMacroBlockReadData &read_info); virtual Type get_type() const { return MACRO_BLOCK_RESTORE_READER; } virtual int64_t get_data_size() const { return data_size_; } @@ -179,10 +195,7 @@ public: ObCopyDDLMacroBlockRestoreReader(); virtual ~ObCopyDDLMacroBlockRestoreReader(); int init(const ObCopyMacroBlockReaderInitParam ¶m); - virtual int get_next_macro_block( - obrpc::ObCopyMacroBlockHeader &header, - blocksstable::ObBufferReader &data, - blocksstable::MacroBlockId ¯o_block_id); + virtual int get_next_macro_block(ObICopyMacroBlockReader::CopyMacroBlockReadData &read_data); virtual Type get_type() const { return DDL_MACRO_BLOCK_RESTORE_READER; } virtual int64_t get_data_size() const { return data_size_; } @@ -883,10 +896,7 @@ public: ObCopyRemoteSSTableMacroBlockRestoreReader(); virtual ~ObCopyRemoteSSTableMacroBlockRestoreReader(); int init(const ObCopyMacroBlockReaderInitParam ¶m); - virtual int get_next_macro_block( - obrpc::ObCopyMacroBlockHeader &header, - blocksstable::ObBufferReader &data, - blocksstable::MacroBlockId ¯o_block_id); + virtual int get_next_macro_block(ObICopyMacroBlockReader::CopyMacroBlockReadData &read_data); virtual Type get_type() const { return REMOTE_SSTABLE_MACRO_BLOCK_RESTORE_READER; } virtual int64_t get_data_size() const { return data_size_; } @@ -900,8 +910,7 @@ private: int read_backup_macro_block_data_( blocksstable::ObDataMacroBlockMeta ¯o_meta, - obrpc::ObCopyMacroBlockHeader &header, - blocksstable::ObBufferReader &data); + ObICopyMacroBlockReader::CopyMacroBlockReadData &read_data); int get_backup_macro_block_index_( const ObRestoreMacroBlockId ¯o_id, backup::ObBackupMacroBlockIndex ¯o_index); @@ -910,8 +919,7 @@ private: blocksstable::ObBufferReader &data_buffer); int read_local_macro_block_data_( blocksstable::ObDataMacroBlockMeta ¯o_meta, - obrpc::ObCopyMacroBlockHeader &header, - blocksstable::ObBufferReader &data); + ObICopyMacroBlockReader::CopyMacroBlockReadData &read_data); private: bool is_inited_; @@ -934,6 +942,7 @@ private: int64_t macro_block_count_; int64_t data_size_; ObSelfBufferWriter meta_row_buf_; + ObMacroBlockReuseMgr *macro_block_reuse_mgr_; DISALLOW_COPY_AND_ASSIGN(ObCopyRemoteSSTableMacroBlockRestoreReader); }; } diff --git a/src/storage/high_availability/ob_storage_ha_struct.cpp b/src/storage/high_availability/ob_storage_ha_struct.cpp index 645d385384..ce9cdf7f77 100644 --- a/src/storage/high_availability/ob_storage_ha_struct.cpp +++ b/src/storage/high_availability/ob_storage_ha_struct.cpp @@ -21,6 +21,7 @@ #include "storage/ls/ob_ls_tablet_service.h" #include "logservice/ob_log_service.h" #include "share/transfer/ob_transfer_task_operator.h" +#include "storage/blocksstable/index_block/ob_sstable_sec_meta_iterator.h" namespace oceanbase { @@ -1984,6 +1985,132 @@ int ObBackfillTabletsTableMgr::get_local_rebuild_seq(int64_t &local_rebuild_seq) return ret; } +/******************ObMacroBlcokReuseMgr*********************/ +ObMacroBlockReuseMgr::ObMacroBlockReuseMgr() + : is_inited_(false), + reuse_maps_() +{ +} + +ObMacroBlockReuseMgr::~ObMacroBlockReuseMgr() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(destroy())) { + LOG_ERROR("failed to destroy macro block reuse mgr", K(ret), K_(is_inited)); + } +} + +int ObMacroBlockReuseMgr::init() +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = MTL_ID(); + + if (is_inited_) { + ret = OB_INIT_TWICE; + LOG_WARN("macro block reuse mgr init twice", K(ret)); + } else if (OB_FAIL(reuse_maps_.init("ReuseMaps", tenant_id))) { + LOG_WARN("failed to init reuse maps", K(ret), K(tenant_id)); + } else { + is_inited_ = true; + LOG_INFO("success to init macro block reuse mgr", K(ret), K(reuse_maps_.is_inited())); + } + + return OB_SUCCESS; +} + +void ObMacroBlockReuseMgr::reset() +{ + int ret = OB_SUCCESS; + + if (!is_inited_) { + LOG_INFO("macro block reuse mgr has not been inited, no need to reset", K_(is_inited)); + } else { + ReuseMaps::BlurredIterator iter(reuse_maps_); + ReuseMajorTableKey reuse_key; + ReuseMajorTableValue *reuse_value = nullptr; + + // destroy all reuse value + while (OB_SUCC(ret)) { + if (OB_FAIL(iter.next(reuse_key, reuse_value))) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } else { + LOG_ERROR("failed to get next reuse value, may cause memory leak!", K(ret)); + } + break; + } else if (OB_ISNULL(reuse_value)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("reuse value is NULL", K(ret), KP(reuse_value)); + } else { + free_reuse_value_(reuse_value); + } + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(reuse_maps_.reset())) { + LOG_WARN("failed to reset reuse maps", K(ret)); + } + } + + if (OB_FAIL(ret)) { + LOG_ERROR("failed to reset macro block reuse mgr, may cause memory leak!!!", K(ret)); + } +} + +int ObMacroBlockReuseMgr::destroy() +{ + int ret = OB_SUCCESS; + + if (!is_inited_) { + LOG_INFO("macro block reuse mgr has not been inited, no need to destroy", K_(is_inited)); + } else { + reset(); + if (OB_FAIL(reuse_maps_.destroy())) { + LOG_WARN("failed to destroy reuse maps", K(ret)); + } else { + is_inited_ = false; + } + } + + return ret; +} + +int ObMacroBlockReuseMgr::count(int64_t &count) +{ + int ret = OB_SUCCESS; + count = 0; + + if (!is_inited_) { + ret = OB_NOT_INIT; + } else { + ReuseMaps::BlurredIterator iter(reuse_maps_); + ReuseMajorTableKey reuse_key; + ReuseMajorTableValue *reuse_value = nullptr; + int64_t tmp_count = 0; + + while (OB_SUCC(ret)) { + if (OB_FAIL(iter.next(reuse_key, reuse_value))) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to get next reuse value", K(ret)); + } + break; + } else if (OB_ISNULL(reuse_value)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("reuse value is NULL", K(ret), KP(reuse_value)); + } else if (OB_FAIL(reuse_value->count(tmp_count))) { + LOG_WARN("fail to count item in single reuse map", K(ret), KPC(reuse_value)); + } else { + count += tmp_count; + } + } + } + + return ret; +} + ObLogicTabletID::ObLogicTabletID() : tablet_id_(), transfer_seq_(-1) @@ -2112,6 +2239,409 @@ int ObMigrationChooseSrcHelperInitParam::assign(const ObMigrationChooseSrcHelper } return ret; } -} + +int ObMacroBlockReuseMgr::get_macro_block_reuse_info( + const ObITable::TableKey &table_key, + const blocksstable::ObLogicMacroBlockId &logic_id, + blocksstable::MacroBlockId ¯o_id, + int64_t &data_checksum) +{ + int ret = OB_SUCCESS; + ReuseMajorTableKey reuse_key; + ReuseMap *reuse_map = nullptr; + int64_t snapshot_version = 0; + int64_t input_version = 0; + MacroBlockReuseInfo reuse_info; + macro_id.reset(); + data_checksum = 0; + + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("macro block reuse mgr do not init", K(ret)); + } else if (!table_key.is_valid() || !logic_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(table_key), K(logic_id)); + } else if (OB_FAIL(get_reuse_key_(table_key, reuse_key))) { + LOG_WARN("failed to get reuse key", K(ret), K(table_key)); + } else if (OB_FAIL(get_reuse_value_(table_key, reuse_map, snapshot_version))) { + LOG_WARN("fail to get reuse value", K(ret), K(table_key)); + } else if (FALSE_IT(input_version = table_key.get_snapshot_version())) { + } else if (snapshot_version >= input_version) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("reuse major snapshot version is equal to or greater than input major snapshot version", K(snapshot_version), K(input_version)); + } else if (OB_FAIL(reuse_map->get(logic_id, reuse_info))) { + LOG_WARN("fail to get reuse info in reuse map", K(ret), K(logic_id), K(table_key)); + } else { + macro_id = reuse_info.id_; + data_checksum = reuse_info.data_checksum_; + } + + return ret; } +int ObMacroBlockReuseMgr::add_macro_block_reuse_info( + const ObITable::TableKey &table_key, + const blocksstable::ObLogicMacroBlockId &logic_id, + const blocksstable::MacroBlockId ¯o_id, + const int64_t &data_checksum) +{ + int ret = OB_SUCCESS; + ReuseMajorTableKey reuse_key; + ReuseMap *reuse_map = nullptr; + MacroBlockReuseInfo reuse_info; + int64_t snapshot_version = 0; + int64_t input_snapshot_version = 0; + + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("macro block reuse mgr do not init", K(ret)); + } else if (!table_key.is_valid() || !logic_id.is_valid() || !macro_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(table_key), K(logic_id)); + } else if (OB_FAIL(get_reuse_key_(table_key, reuse_key))) { + LOG_WARN("failed to get reuse key", K(ret), K(table_key)); + } else if (OB_FAIL(get_reuse_value_(table_key, reuse_map, snapshot_version))) { + LOG_WARN("fail to get reuse value", K(ret), K(table_key)); + } else if (FALSE_IT(input_snapshot_version = table_key.get_snapshot_version())) { + } else if (snapshot_version != input_snapshot_version) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("reuse major snapshot version is not equal to input major snapshot version", K(snapshot_version), K(input_snapshot_version)); + } else if (FALSE_IT(reuse_info.id_ = macro_id)) { + } else if (FALSE_IT(reuse_info.data_checksum_ = data_checksum)) { + } else if (OB_FAIL(reuse_map->insert(logic_id, reuse_info))) { + LOG_WARN("fail to add reuse info into reuse map", K(ret), K(logic_id), K(table_key)); + } + + return ret; +} + +int ObMacroBlockReuseMgr::update_single_reuse_map(const ObITable::TableKey &table_key, const storage::ObTabletHandle &tablet_handle, const blocksstable::ObSSTable &sstable) +{ + int ret = OB_SUCCESS; + ReuseMajorTableKey reuse_key; + int64_t max_snapshot_version = 0; + int64_t input_snapshot_version = 0; + bool need_build = false; + + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("macro block reuse mgr do not init", K(ret)); + } else if (!table_key.is_valid() || !tablet_handle.is_valid() || !sstable.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(table_key), K(tablet_handle), K(sstable)); + } else if (OB_FAIL(get_reuse_key_(table_key, reuse_key))) { + LOG_WARN("failed to get reuse key", K(ret), K(table_key)); + } else if (OB_FAIL(get_major_snapshot_version(table_key, max_snapshot_version))) { + if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_SUCCESS; + need_build = true; + LOG_INFO("major not in reuse mgr, no need to remove", K(ret), K(need_build), K(table_key)); + } else { + LOG_WARN("failed to get major snapshot version in mgr", K(ret), K(table_key)); + } + } else if (FALSE_IT(input_snapshot_version = table_key.get_snapshot_version())) { + } else if (max_snapshot_version >= table_key.get_snapshot_version()) { + LOG_INFO("major snapshot version of mgr is equal to or greater than input snapshot version, no need to build", K(need_build), K(max_snapshot_version), K(input_snapshot_version), K(table_key)); + } else if (OB_FAIL(remove_single_reuse_map_(reuse_key))) { + LOG_INFO("failed to remove reuse map", K(ret), K(reuse_key)); + } else { + need_build = true; + LOG_INFO("major snapshot version of mgr is less than input snapshot version, remove old reuse map then build", K(need_build), K(max_snapshot_version), K(input_snapshot_version), K(table_key)); + } + + if (OB_SUCC(ret) && need_build) { + LOG_INFO("build reuse map for major sstable", K(ret), K(need_build), K(table_key)); + if (OB_FAIL(build_single_reuse_map_(table_key, tablet_handle, sstable))) { + LOG_WARN("failed to build reuse map", K(ret), K(table_key)); + } else { + LOG_INFO("success to update reuse map", K(ret), K(max_snapshot_version), K(input_snapshot_version), K(table_key)); + } + } + + return ret; +} + +int ObMacroBlockReuseMgr::get_major_snapshot_version(const ObITable::TableKey &table_key, int64_t &snapshot_version) +{ + int ret = OB_SUCCESS; + ReuseMap *reuse_map = nullptr; + snapshot_version = 0; + + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("macro block reuse mgr do not init", K(ret)); + } else if (!table_key.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid table key", K(ret), K(table_key)); + } else if (OB_FAIL(get_reuse_value_(table_key, reuse_map, snapshot_version))) { + LOG_WARN("fail to get reuse value", K(ret), K(table_key)); + } + + return ret; +} + +ObMacroBlockReuseMgr::MacroBlockReuseInfo::MacroBlockReuseInfo() + : id_(), + data_checksum_(0) +{ +} + +void ObMacroBlockReuseMgr::MacroBlockReuseInfo::reset() +{ + id_.reset(); + data_checksum_ = 0; +} + +ObMacroBlockReuseMgr::ReuseMajorTableKey::ReuseMajorTableKey() + : tablet_id_(0), + column_group_idx_(0) +{ +} + +ObMacroBlockReuseMgr::ReuseMajorTableKey::ReuseMajorTableKey( + const common::ObTabletID &tablet_id, + const uint16_t column_group_idx) + : tablet_id_(tablet_id), + column_group_idx_(column_group_idx) +{ +} + +void ObMacroBlockReuseMgr::ReuseMajorTableKey::reset() +{ + tablet_id_.reset(); + column_group_idx_ = 0; +} + +uint64_t ObMacroBlockReuseMgr::ReuseMajorTableKey::hash() const +{ + uint64_t hash_val = 0; + hash_val = tablet_id_.hash(); + hash_val = common::murmurhash(&column_group_idx_, sizeof(column_group_idx_), hash_val); + return hash_val; +} + +bool ObMacroBlockReuseMgr::ReuseMajorTableKey::operator == (const ReuseMajorTableKey &other) const +{ + return tablet_id_ == other.tablet_id_ && column_group_idx_ == other.column_group_idx_; +} + +ObMacroBlockReuseMgr::ReuseMajorTableValue::ReuseMajorTableValue() + : is_inited_(false), + snapshot_version_(0), + reuse_map_() +{ +} + +ObMacroBlockReuseMgr::ReuseMajorTableValue::~ReuseMajorTableValue() +{ + if (is_inited_) { + reuse_map_.destroy(); + is_inited_ = false; + } +} + +int ObMacroBlockReuseMgr::ReuseMajorTableValue::init(const int64_t &snapshot_version) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = MTL_ID(); + + if (is_inited_) { + ret = OB_INIT_TWICE; + LOG_WARN("reuse major table value init twice", K(ret)); + } else if (snapshot_version < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid snapshot version", K(ret), K(snapshot_version)); + } else if (OB_FAIL(reuse_map_.init("ReuseMap", tenant_id))) { + LOG_WARN("failed to init reuse map", K(ret), K(tenant_id)); + } else { + snapshot_version_ = snapshot_version; + is_inited_ = true; + } + + return ret; +} + +int ObMacroBlockReuseMgr::ReuseMajorTableValue::count(int64_t &count) +{ + int ret = OB_SUCCESS; + + count = 0; + if (!is_inited_) { + ret = OB_NOT_INIT; + } else { + count = reuse_map_.count(); + } + + return ret; +} + +int ObMacroBlockReuseMgr::get_reuse_key_(const ObITable::TableKey &table_key, ReuseMajorTableKey &reuse_key) +{ + int ret = OB_SUCCESS; + reuse_key.reset(); + + if (!table_key.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid table key",K(ret), K(table_key)); + } else { + reuse_key.tablet_id_ = table_key.tablet_id_; + reuse_key.column_group_idx_ = table_key.column_group_idx_; + } + + return ret; +} + +int ObMacroBlockReuseMgr::get_reuse_value_(const ObITable::TableKey &table_key, ReuseMap *&reuse_map, int64_t &snapshot_version) +{ + int ret = OB_SUCCESS; + ReuseMajorTableKey reuse_key; + ReuseMajorTableValue *reuse_value = nullptr; + snapshot_version = 0; + + if (OB_FAIL(get_reuse_key_(table_key, reuse_key))) { + LOG_WARN("failed to get reuse key", K(ret), K(table_key)); + } else if (OB_FAIL(reuse_maps_.get(reuse_key, reuse_value))) { + LOG_WARN("failed to get reuse value", K(ret), K(reuse_key)); + } else if (OB_ISNULL(reuse_value)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("reuse value is null", K(ret), KP(reuse_value)); + } else if (!reuse_value->is_inited_){ + ret = OB_NOT_INIT; + LOG_WARN("reuse value is not inited", K(ret), KPC(reuse_value)); + } else { + reuse_map = &reuse_value->reuse_map_; + snapshot_version = reuse_value->snapshot_version_; + } + + return ret; +} + +int ObMacroBlockReuseMgr::remove_single_reuse_map_(const ReuseMajorTableKey &reuse_key) +{ + int ret = OB_SUCCESS; + ReuseMajorTableValue *reuse_value = nullptr; + + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("macro block reuse mgr do not init", K(ret)); + } else if (OB_FAIL(reuse_maps_.erase(reuse_key, reuse_value))) { + LOG_WARN("failed to remove reuse map", K(ret), K(reuse_key)); + } else if (OB_ISNULL(reuse_value)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("reuse value is null", K(ret), K(reuse_key), KP(reuse_value)); + } else { + free_reuse_value_(reuse_value); + } + + return ret; +} + +int ObMacroBlockReuseMgr::build_single_reuse_map_(const ObITable::TableKey &table_key, const storage::ObTabletHandle &tablet_handle, const blocksstable::ObSSTable &sstable) +{ + int ret = OB_SUCCESS; + ObArenaAllocator allocator; + ObDatumRange datum_range; + const storage::ObITableReadInfo *index_read_info = nullptr; + + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("macro block reuse mgr do not init", K(ret)); + } else if (!table_key.is_valid() || !tablet_handle.is_valid() || !sstable.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), K(table_key), K(tablet_handle), K(sstable)); + } else { + SMART_VAR(ObSSTableSecMetaIterator, meta_iter) { + if (FALSE_IT(datum_range.set_whole_range())) { + } else if (OB_FAIL(tablet_handle.get_obj()->get_sstable_read_info(&sstable, index_read_info))) { + LOG_WARN("failed to get index read info ", KR(ret), K(sstable)); + } else if (OB_FAIL(meta_iter.open(datum_range, + ObMacroBlockMetaType::DATA_BLOCK_META, + sstable, + *index_read_info, + allocator))) { + LOG_WARN("failed to open sec meta iterator", K(ret)); + } else { + ObDataMacroBlockMeta data_macro_block_meta; + ObLogicMacroBlockId logic_id; + MacroBlockId macro_id; + ReuseMajorTableKey reuse_key; + ReuseMajorTableValue *reuse_value = nullptr; + + if (OB_FAIL(get_reuse_key_(table_key, reuse_key))) { + LOG_WARN("failed to get reuse key", K(ret), K(table_key)); + } else if (OB_FAIL(prepare_reuse_value_(table_key.get_snapshot_version(), reuse_value))) { + LOG_WARN("failed to init reuse value", K(ret), K(table_key)); + } else { + while (OB_SUCC(ret)) { + data_macro_block_meta.reset(); + logic_id.reset(); + if (OB_FAIL(meta_iter.get_next(data_macro_block_meta))) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + break; + } else { + LOG_WARN("failed to get next", K(ret)); + } + } else { + MacroBlockReuseInfo reuse_info; + logic_id = data_macro_block_meta.get_logic_id(); + reuse_info.id_ = data_macro_block_meta.get_macro_id(); + reuse_info.data_checksum_ = data_macro_block_meta.get_meta_val().data_checksum_; + + if (OB_FAIL(reuse_value->reuse_map_.insert(logic_id, reuse_info))) { + LOG_WARN("failed to insert reuse info into reuse map", K(ret), K(logic_id), K(reuse_info), K(table_key)); + } + } + } + + if (OB_FAIL(ret)) { + LOG_WARN("failed to build reuse value, destory it", K(ret), K(table_key)); + free_reuse_value_(reuse_value); + } else if (OB_FAIL(reuse_maps_.insert(reuse_key, reuse_value))) { + LOG_WARN("failed to set reuse map, destroy reuse value", K(ret), K(reuse_key)); + free_reuse_value_(reuse_value); + } + } + } + } + } + + return ret; +} + +int ObMacroBlockReuseMgr::prepare_reuse_value_(const int64_t &snapshot_version, ReuseMajorTableValue *&reuse_value) +{ + int ret = OB_SUCCESS; + void *buf = nullptr; + ReuseMajorTableValue *tmp_value = nullptr; + reuse_value = nullptr; + + if (OB_ISNULL(buf = mtl_malloc(sizeof(ReuseMajorTableValue), "ReuseValue"))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret), KP(buf)); + } else if (FALSE_IT(tmp_value = new (buf) ReuseMajorTableValue())) { + } else if (OB_FAIL(tmp_value->init(snapshot_version))) { + LOG_WARN("failed to init reuse value", K(ret), K(snapshot_version)); + } else { + reuse_value = tmp_value; + tmp_value = nullptr; + } + + if (OB_NOT_NULL(tmp_value)) { + free_reuse_value_(tmp_value); + } + + return ret; +} + +void ObMacroBlockReuseMgr::free_reuse_value_(ReuseMajorTableValue *&reuse_value) +{ + if (OB_NOT_NULL(reuse_value)) { + reuse_value->~ReuseMajorTableValue(); + mtl_free(reuse_value); + reuse_value = nullptr; + } +} + +} +} \ No newline at end of file diff --git a/src/storage/high_availability/ob_storage_ha_struct.h b/src/storage/high_availability/ob_storage_ha_struct.h index 3460921f79..db5914a9ef 100644 --- a/src/storage/high_availability/ob_storage_ha_struct.h +++ b/src/storage/high_availability/ob_storage_ha_struct.h @@ -541,6 +541,114 @@ private: DISALLOW_COPY_AND_ASSIGN(ObBackfillTabletsTableMgr); }; +class ObMacroBlockReuseMgr final +{ +public: + ObMacroBlockReuseMgr(); + ~ObMacroBlockReuseMgr(); + int init(); + void reset(); + int destroy(); + int count(int64_t &count); + bool is_inited() const { return is_inited_; } + // get the macro block physical ID and data checksum by the macro block logical ID + int get_macro_block_reuse_info( + const ObITable::TableKey &table_key, + const blocksstable::ObLogicMacroBlockId &logic_id, + blocksstable::MacroBlockId ¯o_id, + int64_t &data_checksum); + // add the macro block logical ID -> [physical ID, data checksum] and data checksum mapping + int add_macro_block_reuse_info( + const ObITable::TableKey &table_key, + const blocksstable::ObLogicMacroBlockId &logic_id, + const blocksstable::MacroBlockId ¯o_id, + const int64_t &data_checksum); + // update single reuse map of the chosen major sstable + // if the snapshot version of the input table key is larger than the one in the reuse map, update the reuse map + int update_single_reuse_map(const ObITable::TableKey &table_key, const storage::ObTabletHandle &tablet_handle, const blocksstable::ObSSTable &sstable); + // get target sstable's version in the reuse map + int get_major_snapshot_version(const ObITable::TableKey &table_key, int64_t &snapshot_version); +public: + static int64_t get_item_size() { + // size of key + size of value + size pointer of next node (linear hash map) + // see ObLinearHashMap::Node + return sizeof(blocksstable::ObLogicMacroBlockId) + + sizeof(MacroBlockReuseInfo) + + sizeof(void *); + } +private: + // physical ID and data checksum of a macro block, value of a single reuse map + struct MacroBlockReuseInfo final + { + public: + MacroBlockReuseInfo(); + ~MacroBlockReuseInfo() = default; + void reset(); + public: + blocksstable::MacroBlockId id_; + int64_t data_checksum_; + + TO_STRING_KV( + K_(id), + K_(data_checksum)); + }; + // logical ID -> [physical ID, data checksum] mapping of a major sstable. + typedef ObLinearHashMap ReuseMap; + // Key of the reuse_maps, use the tablet_id and column_group_idx to identify the lastest local snapshot version + // and the reuse info (logical ID -> [physical ID, data checksum] mapping) of a major sstable. + struct ReuseMajorTableKey final + { + public: + ReuseMajorTableKey(); + ReuseMajorTableKey(const common::ObTabletID &tablet_id, const uint16_t column_group_idx); + ~ReuseMajorTableKey() = default; + void reset(); + uint64_t hash() const; + int hash(uint64_t &hash_val) const { hash_val = hash(); return OB_SUCCESS; }; + bool operator == (const ReuseMajorTableKey &other) const; + TO_STRING_KV( + K_(tablet_id), + K_(column_group_idx)); + + public: + common::ObTabletID tablet_id_; + uint16_t column_group_idx_; + }; + // Value of the reuse_maps, indicate the reuse info (logical ID -> [physical ID, data checksum] mapping) of a + // specific version major sstable (the latest local snapshot version). + struct ReuseMajorTableValue final + { + public: + ReuseMajorTableValue(); + ~ReuseMajorTableValue(); + int init(const int64_t &snapshot_version); + int count(int64_t &count); + TO_STRING_KV( + K_(is_inited), + K_(snapshot_version)); + public: + bool is_inited_; + int64_t snapshot_version_; + ReuseMap reuse_map_; + }; + typedef ObLinearHashMap ReuseMaps; +private: + int get_reuse_key_(const ObITable::TableKey &table_key, ReuseMajorTableKey &reuse_key); + int get_reuse_value_(const ObITable::TableKey &table_key, ReuseMap *&reuse_map, int64_t &snapshot_version); + // remove single reuse map of the chosen major sstable (chosen by table_key) + int remove_single_reuse_map_(const ReuseMajorTableKey &reuse_key); + // build single reuse map of the chosen major sstable + int build_single_reuse_map_(const ObITable::TableKey &table_key, const storage::ObTabletHandle &tablet_handle, const blocksstable::ObSSTable &sstable); + // alloc reuse value then init it + int prepare_reuse_value_(const int64_t &snapshot_version, ReuseMajorTableValue *&reuse_value); + // free reuse value + void free_reuse_value_(ReuseMajorTableValue *&reuse_value); +private: + bool is_inited_; + ReuseMaps reuse_maps_; // mapping from major sstable to reuse info (if not column store, this map will only contain one element) + DISALLOW_COPY_AND_ASSIGN(ObMacroBlockReuseMgr); +}; + struct ObLogicTabletID final { public: diff --git a/src/storage/high_availability/ob_storage_ha_utils.cpp b/src/storage/high_availability/ob_storage_ha_utils.cpp index 034e6858a7..ad3e0ecfaf 100644 --- a/src/storage/high_availability/ob_storage_ha_utils.cpp +++ b/src/storage/high_availability/ob_storage_ha_utils.cpp @@ -1493,5 +1493,173 @@ void ObTransferUtils::transfer_tablet_restore_stat( } } +int ObStorageHAUtils::build_major_sstable_reuse_info( + const ObTabletHandle &tablet_handle, + ObMacroBlockReuseMgr ¯o_block_reuse_mgr, + const bool &is_restore) +{ + // 1. get local max major sstable snapshot version (and related sstable) + // 2. iterate these major sstables' macro blocks (if not co, there is only one major sstable), update reuse map + int ret = OB_SUCCESS; + ObTablet *tablet = nullptr; + ObITable *latest_major = nullptr; + ObTabletMemberWrapper wrapper; + int64_t major_cnt = 0; + + if (!tablet_handle.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(tablet_handle)); + } else if (FALSE_IT(tablet = tablet_handle.get_obj())) { + } else { + if (macro_block_reuse_mgr.is_inited()) { + LOG_INFO("reuse info mgr has been inited before (maybe retry), won't init again", K(macro_block_reuse_mgr.is_inited())); + } else if (OB_FAIL(macro_block_reuse_mgr.init())) { + LOG_WARN("failed to init reuse info mgr", K(ret)); + } + + // when restore, we won't build reuse info for lastest major sstable, because restore always read all sstable from backup media + // (i.e. will scan all sstables' macro block again when tablet restore dag retry) + // when migrate, we will keep the major sstable already been copied to dest server, so we need to build reuse info for lastest major sstable + // that already been copied to dest server + if (OB_SUCC(ret) && !is_restore) { + common::ObArray major_sstables; + int64_t reuse_info_count = 0; + + if (OB_FAIL(tablet->fetch_table_store(wrapper))) { + LOG_WARN("failed to fetch table store", K(ret), KPC(tablet)); + } else if (!wrapper.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("table store wrapper is invalid", K(ret), K(wrapper), KPC(tablet)); + } else if (FALSE_IT(major_cnt = wrapper.get_member()->get_major_sstables().count())) { + } else if (0 == major_cnt) { + LOG_INFO("no major sstable, skip build reuse info", K(ret), K(wrapper), KPC(tablet)); + } else if (OB_FAIL(get_latest_available_major_(wrapper.get_member()->get_major_sstables(), latest_major))) { + // get_major_sstables return major sstables ordered by snapshot version in ascending order + LOG_WARN("failed to get latest available major sstable", K(ret), K(wrapper), KPC(tablet)); + } else if (OB_ISNULL(latest_major)) { + // skip, first major sstable has backup data, no need to build reuse info + LOG_INFO("first major sstable has backup data, no need to build reuse info", K(ret), K(wrapper), KPC(tablet)); + } else if (OB_FAIL(get_latest_major_sstable_array_(latest_major, major_sstables))){ + LOG_WARN("failed to get latest major sstable array", K(ret), KPC(latest_major)); + } else { + if (OB_FAIL(build_reuse_info_(major_sstables, tablet_handle, macro_block_reuse_mgr))) { + LOG_WARN("failed to build reuse info", K(ret), K(major_sstables), KPC(tablet), KPC(latest_major)); + } else if (OB_FAIL(macro_block_reuse_mgr.count(reuse_info_count))) { + LOG_WARN("failed to count reuse info", K(ret), K(major_sstables), KPC(tablet), KPC(latest_major)); + } else { + LOG_INFO("succeed to build reuse info", K(ret), K(major_sstables), KPC(tablet), KPC(latest_major), K(reuse_info_count)); + } + + // if build reuse info failed, reset reuse mgr + if (OB_FAIL(ret)) { + macro_block_reuse_mgr.reset(); + } + } + } + } + + return ret; +} + +int ObStorageHAUtils::get_latest_available_major_(const storage::ObSSTableArray &major_sstables, ObITable *&latest_major) +{ + int ret = OB_SUCCESS; + latest_major = nullptr; + + // major sstables must be sorted by snapshot version in ascending order + // get the latest major sstable that has no backup data and all previous major sstables have backup data + for(int64_t i = 0; OB_SUCC(ret) && i < major_sstables.count(); ++i) { + ObITable *cur_major = major_sstables.at(i); + ObSSTable *sstable = nullptr; + ObSSTableMetaHandle sst_meta_hdl; + + if (OB_ISNULL(cur_major) || !ObITable::is_major_sstable(cur_major->get_key().table_type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid major sstable", K(ret), K(i), KPC(cur_major)); + } else if (FALSE_IT(sstable = static_cast(cur_major))) { + } else if (OB_FAIL(sstable->get_meta(sst_meta_hdl))) { + LOG_WARN("failed to get sstable meta", K(ret), KPC(sstable)); + } else if (sst_meta_hdl.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup()) { + // stop at the first major sstable that has backup data + break; + } else { + latest_major = cur_major; + } + } + + return ret; +} + +int ObStorageHAUtils::get_latest_major_sstable_array_(const ObITable *latest_major, common::ObArray &major_sstables) +{ + int ret = OB_SUCCESS; + ObITable::TableKey table_key = latest_major->get_key(); + // table type of the local major sstable which has max snapshot version + // could be normal major sstable (row store) or co sstable (column store) + ObITable::TableType table_type = table_key.table_type_; + + if (table_type != ObITable::COLUMN_ORIENTED_SSTABLE && table_type != ObITable::MAJOR_SSTABLE) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid table type", K(ret), K(table_type)); + } else if (table_type == ObITable::MAJOR_SSTABLE) { + const ObSSTable *sstable = static_cast (latest_major); + if (!ObITable::is_major_sstable(sstable->get_key().table_type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid sstable type, not major sstable", K(ret), KPC(sstable)); + } else if(OB_FAIL(major_sstables.push_back(sstable))) { + LOG_WARN("failed to push back sstable", K(ret), KPC(sstable)); + } + } else if (table_type == ObITable::COLUMN_ORIENTED_SSTABLE) { + const ObCOSSTableV2 *co_sstable = static_cast (latest_major); + ObArray sstable_wrappers; + + if (OB_FAIL(co_sstable->get_all_tables(sstable_wrappers))) { + LOG_WARN("failed to get all co & cg tables", K(ret), K(table_key), KPC(latest_major)); + } else { + ObSSTable *sstable = nullptr; + + // add all cg sstable and the lastest co sstable to build reuse info + for (int64_t i = 0; OB_SUCC(ret) && i < sstable_wrappers.count(); ++i) { + sstable = sstable_wrappers.at(i).get_sstable(); + + if (OB_ISNULL(sstable)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("sstable is null", K(ret), KP(sstable)); + } else if (!sstable->is_column_store_sstable()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid sstable type, not column store sstable", K(ret), KPC(sstable)); + } else if (OB_FAIL(major_sstables.push_back(sstable))) { + LOG_WARN("failed to push back sstable", K(ret), KPC(sstable)); + } + } + } + } + + return ret; +} + +int ObStorageHAUtils::build_reuse_info_(const common::ObArray &major_sstables, const ObTabletHandle &tablet_handle, ObMacroBlockReuseMgr ¯o_block_reuse_mgr) +{ + int ret = OB_SUCCESS; + + for (int64_t i = 0; OB_SUCC(ret) && i < major_sstables.count(); ++i) { + const ObSSTable *sstable = major_sstables.at(i); + if (OB_ISNULL(sstable) || !ObITable::is_major_sstable(sstable->get_key().table_type_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("sstable should not be NULL and should be major" , K(ret), KPC(sstable)); + } else if (OB_FAIL(macro_block_reuse_mgr.update_single_reuse_map(sstable->get_key(), tablet_handle, *sstable))) { + LOG_WARN("failed to update reuse map", K(ret), K(tablet_handle), KPC(sstable)); + } + } + + return ret; +} + +void ObStorageHAUtils::sort_table_key_array_by_snapshot_version(common::ObArray &table_key_array) +{ + TableKeySnapshotVersionComparator cmp; + lib::ob_sort(table_key_array.begin(), table_key_array.end(), cmp); +} + } // end namespace storage } // end namespace oceanbase diff --git a/src/storage/high_availability/ob_storage_ha_utils.h b/src/storage/high_availability/ob_storage_ha_utils.h index a39ae16b1e..b3f3a0d213 100644 --- a/src/storage/high_availability/ob_storage_ha_utils.h +++ b/src/storage/high_availability/ob_storage_ha_utils.h @@ -81,8 +81,19 @@ public: static int append_tablet_list( const common::ObIArray &logic_tablet_id_array, common::ObIArray &tablet_id_array); + static int build_major_sstable_reuse_info( + const ObTabletHandle &tablet_handle, + ObMacroBlockReuseMgr ¯o_block_reuse_mgr, + const bool &is_restore); + static void sort_table_key_array_by_snapshot_version(common::ObArray &table_key_array); static int get_tablet_size_in_bytes(const ObLSID &ls_id, const ObTabletID &tablet_id, int64_t &tablet_size); private: + struct TableKeySnapshotVersionComparator final + { + bool operator()(const ObITable::TableKey &lhs, const ObITable::TableKey &rhs) { + return lhs.get_snapshot_version() < rhs.get_snapshot_version(); + } + }; static int check_merge_error_(const uint64_t tenant_id, common::ObISQLClient &sql_client); static int fetch_src_tablet_meta_info_(const uint64_t tenant_id, const common::ObTabletID &tablet_id, const share::ObLSID &ls_id, const common::ObAddr &src_addr, common::ObISQLClient &sql_client, @@ -90,6 +101,12 @@ private: static int check_tablet_replica_checksum_(const uint64_t tenant_id, const common::ObTabletID &tablet_id, const share::ObLSID &ls_id, const share::SCN &compaction_scn, common::ObISQLClient &sql_client); static int get_readable_scn_(share::SCN &readable_scn); + static int get_latest_major_sstable_array_(const ObITable *latest_major, common::ObArray &major_sstables); + static int build_reuse_info_( + const common::ObArray &major_sstabls, + const ObTabletHandle &tablet_handle, + ObMacroBlockReuseMgr ¯o_block_reuse_mgr); + static int get_latest_available_major_(const storage::ObSSTableArray & major_sstables, ObITable *&latest_major); }; struct ObTransferUtils diff --git a/src/storage/high_availability/ob_tablet_copy_finish_task.cpp b/src/storage/high_availability/ob_tablet_copy_finish_task.cpp index 3bc9f32d17..b97bea7617 100644 --- a/src/storage/high_availability/ob_tablet_copy_finish_task.cpp +++ b/src/storage/high_availability/ob_tablet_copy_finish_task.cpp @@ -44,6 +44,7 @@ ObTabletCopyFinishTask::ObTabletCopyFinishTask() minor_tables_handle_(), ddl_tables_handle_(), major_tables_handle_(), + mds_tables_handle_(), restore_action_(ObTabletRestoreAction::MAX), src_tablet_meta_(nullptr), copy_tablet_ctx_(nullptr), @@ -99,6 +100,7 @@ int ObTabletCopyFinishTask::process() int ret = OB_SUCCESS; bool only_contain_major = false; ObCopyTabletStatus::STATUS status = ObCopyTabletStatus::MAX_STATUS; + const ObCopyTabletRecordExtraInfo *extra_info = nullptr; if (!is_inited_) { ret = OB_NOT_INIT; @@ -134,13 +136,24 @@ int ObTabletCopyFinishTask::process() + ddl_tables_handle_.get_count() + major_tables_handle_.get_count(); + int tmp_ret = OB_SUCCESS; + char extra_info_str[MAX_ROOTSERVICE_EVENT_EXTRA_INFO_LENGTH] = {0}; + if (OB_SUCCESS != (tmp_ret = copy_tablet_ctx_->get_copy_tablet_record_extra_info(extra_info))) { + LOG_WARN("failed to get copy tablet record extra info", K(tmp_ret), KP(extra_info)); + } else if (OB_ISNULL(extra_info)) { + LOG_WARN("copy tablet record extra info is NULL", K(extra_info)); + } else if (OB_SUCCESS != (tmp_ret = common::databuff_printf(extra_info_str, MAX_ROOTSERVICE_EVENT_EXTRA_INFO_LENGTH, "%s", to_cstring(*extra_info)))) { + LOG_WARN("failed to print extra info", K(tmp_ret), K(extra_info)); + } + SERVER_EVENT_ADD("storage_ha", "tablet_copy_finish_task", "tenant_id", MTL_ID(), "ls_id", ls_->get_ls_id().id(), "tablet_id", tablet_id_.id(), "ret", ret, "result", ha_dag_->get_ha_dag_net_ctx()->is_failed(), - "sstable_count", sstable_count); + "sstable_count", sstable_count, + extra_info_str); if (OB_FAIL(ret)) { int tmp_ret = OB_SUCCESS; @@ -574,6 +587,51 @@ int ObTabletCopyFinishTask::get_mds_sstable_max_end_scn_(share::SCN &max_end_scn return ret; } +/******************ObCopyTabletRecordExtraInfo*********************/ +ObCopyTabletRecordExtraInfo::ObCopyTabletRecordExtraInfo() + : cost_time_ms_(0), + total_data_size_(0), + write_data_size_(0), + major_count_(0), + macro_count_(0), + major_macro_count_(0), + reuse_macro_count_(0), + max_reuse_mgr_size_(0) +{ +} + +ObCopyTabletRecordExtraInfo::~ObCopyTabletRecordExtraInfo() +{ +} + +void ObCopyTabletRecordExtraInfo::reset() +{ + cost_time_ms_ = 0; + total_data_size_ = 0; + write_data_size_ = 0; + major_count_ = 0; + macro_count_ = 0; + major_macro_count_ = 0; + reuse_macro_count_ = 0; + max_reuse_mgr_size_ = 0; +} + +int ObCopyTabletRecordExtraInfo::update_max_reuse_mgr_size(ObMacroBlockReuseMgr *&reuse_mgr) +{ + int ret = OB_SUCCESS; + int64_t count = 0; + + if (OB_ISNULL(reuse_mgr)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("update max reuse mgr size get invalid argument", K(ret), KP(reuse_mgr)); + } else if (OB_FAIL(reuse_mgr->count(count))) { + LOG_WARN("failed to count reuse mgr", K(ret), KP(reuse_mgr)); + } else { + max_reuse_mgr_size_ = MAX(count * reuse_mgr->get_item_size(), max_reuse_mgr_size_); + } + + return ret; +} } } diff --git a/src/storage/high_availability/ob_tablet_group_restore.cpp b/src/storage/high_availability/ob_tablet_group_restore.cpp index e34285a791..4c104564f7 100644 --- a/src/storage/high_availability/ob_tablet_group_restore.cpp +++ b/src/storage/high_availability/ob_tablet_group_restore.cpp @@ -142,6 +142,8 @@ ObTabletRestoreCtx::ObTabletRestoreCtx() ha_table_info_mgr_(nullptr), need_check_seq_(false), ls_rebuild_seq_(-1), + macro_block_reuse_mgr_(), + extra_info_(), lock_(common::ObLatchIds::RESTORE_LOCK), status_(ObCopyTabletStatus::MAX_STATUS) { @@ -181,6 +183,7 @@ void ObTabletRestoreCtx::reset() ls_rebuild_seq_ = -1; status_ = ObCopyTabletStatus::MAX_STATUS; ha_table_info_mgr_ = nullptr; + extra_info_.reset(); } int ObTabletRestoreCtx::set_copy_tablet_status(const ObCopyTabletStatus::STATUS &status) @@ -210,6 +213,18 @@ int ObTabletRestoreCtx::get_copy_tablet_status(ObCopyTabletStatus::STATUS &statu return ret; } +int ObTabletRestoreCtx::get_copy_tablet_record_extra_info(const ObCopyTabletRecordExtraInfo *&extra_info) const +{ + int ret = OB_SUCCESS; + if (!is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tablet restore ctx is invalid", K(ret), KPC(this)); + } else { + extra_info = &extra_info_; + } + return ret; +} + /******************ObTabletGroupRestoreDagNet*********************/ ObTGRDagNetInitParam::ObTGRDagNetInitParam() : arg_(), @@ -766,6 +781,26 @@ int ObInitialTabletGroupRestoreTask::process() "ls_id", ctx_->arg_.ls_id_); DEBUG_SYNC(BEFORE_LEADER_RESTORE_GROUP_TABLET); } +#ifdef ERRSIM + if (ObTabletRestoreAction::is_restore_replace_remote_sstable(ctx_->arg_.action_)) { + logservice::ObLogService *log_srv = nullptr; + ObRole role = ObRole::INVALID_ROLE; + int64_t proposal_id = 0; + + SERVER_EVENT_SYNC_ADD("storage_ha", "before_follower_replace_remote_sstable", + "ls_id", ctx_->arg_.ls_id_); + LOG_INFO("[ERRSIM] before replace remote sstable", KPC(ctx_)); + + if (OB_ISNULL(log_srv = MTL(logservice::ObLogService *))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("[ERRSIM] log service should not be NULL", K(ret), KP(log_srv)); + } else if (OB_FAIL(log_srv->get_palf_role(ctx_->arg_.ls_id_, role, proposal_id))) { + LOG_WARN("[ERRSIM] failed to get palf role", K(ret), KPC(ctx_)); + } else if (is_follower(role)) { + DEBUG_SYNC(BEFORE_FOLLOWER_REPLACE_REMOTE_SSTABLE); + } + } +#endif if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("initial tablet group restore task do not init", K(ret)); @@ -1957,6 +1992,8 @@ int ObTabletRestoreDag::inner_reset_status_for_retry() if (OB_SUCC(ret)) { if (OB_SUCC(ret)) { tablet_restore_ctx_.tablet_handle_.reset(); + tablet_restore_ctx_.extra_info_.reset(); + tablet_restore_ctx_.macro_block_reuse_mgr_.reset(); if (OB_ISNULL(ls = ls_handle_.get_ls())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls should not be NULL", K(ret), K(tablet_restore_ctx_)); @@ -2156,6 +2193,8 @@ int ObTabletRestoreTask::process() if (OB_FAIL(update_ha_status_(status))) { LOG_WARN("failed to update ha status", K(ret), KPC(tablet_restore_ctx_)); } + } else if (OB_FAIL(ObStorageHAUtils::build_major_sstable_reuse_info(tablet_restore_ctx_->tablet_handle_, tablet_restore_ctx_->macro_block_reuse_mgr_, true /* is_restore */))) { + LOG_WARN("failed to build major sstable reuse info", K(ret), KPC(tablet_restore_ctx_)); } else if (OB_FAIL(generate_restore_tasks_())) { LOG_WARN("failed to generate restore tasks", K(ret), KPC(tablet_restore_ctx_)); } @@ -2312,6 +2351,8 @@ int ObTabletRestoreTask::generate_physical_restore_task_( } else if (FALSE_IT(init_param.need_sort_macro_meta_ = !copy_table_key.is_normal_cg_sstable())) { } else if (FALSE_IT(init_param.need_check_seq_ = tablet_restore_ctx_->need_check_seq_)) { } else if (FALSE_IT(init_param.ls_rebuild_seq_ = tablet_restore_ctx_->ls_rebuild_seq_)) { + } else if (FALSE_IT(init_param.macro_block_reuse_mgr_ = ObITable::is_major_sstable(copy_table_key.table_type_) ? &tablet_restore_ctx_->macro_block_reuse_mgr_ : nullptr)) { + } else if (FALSE_IT(init_param.extra_info_ = &tablet_restore_ctx_->extra_info_)) { } else if (OB_FAIL(tablet_restore_ctx_->ha_table_info_mgr_->get_table_info(tablet_restore_ctx_->tablet_id_, copy_table_key, init_param.sstable_param_))) { LOG_WARN("failed to get table info", K(ret), KPC(tablet_restore_ctx_), K(copy_table_key)); @@ -2475,6 +2516,9 @@ int ObTabletRestoreTask::build_copy_table_key_info_() } else if (OB_FAIL(tablet_restore_ctx_->ha_table_info_mgr_->get_table_keys( tablet_restore_ctx_->tablet_id_, copy_table_key_array_))) { LOG_WARN("failed to get copy table keys", K(ret), KPC(tablet_restore_ctx_)); + } else if (FALSE_IT(ObStorageHAUtils::sort_table_key_array_by_snapshot_version(copy_table_key_array_))) { + } else { + LOG_INFO("succeed to build copy table key info", K(copy_table_key_array_)); } return ret; } diff --git a/src/storage/high_availability/ob_tablet_group_restore.h b/src/storage/high_availability/ob_tablet_group_restore.h index 1aa77b62fb..17a6d9e07a 100644 --- a/src/storage/high_availability/ob_tablet_group_restore.h +++ b/src/storage/high_availability/ob_tablet_group_restore.h @@ -88,6 +88,7 @@ public: void reset(); int set_copy_tablet_status(const ObCopyTabletStatus::STATUS &status) override; int get_copy_tablet_status(ObCopyTabletStatus::STATUS &status) const override; + int get_copy_tablet_record_extra_info(const ObCopyTabletRecordExtraInfo *&extra_info) const override; VIRTUAL_TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(tablet_id), KPC_(restore_base_info), K_(is_leader), K_(action), KP_(meta_index_store), KP_(second_meta_index_store), K_(replica_type), KP_(ha_table_info_mgr), K_(status)); @@ -105,6 +106,8 @@ public: ObStorageHATableInfoMgr *ha_table_info_mgr_; bool need_check_seq_; int64_t ls_rebuild_seq_; + ObMacroBlockReuseMgr macro_block_reuse_mgr_; + ObCopyTabletRecordExtraInfo extra_info_; private: common::SpinRWLock lock_; ObCopyTabletStatus::STATUS status_; diff --git a/src/storage/high_availability/ob_transfer_handler.cpp b/src/storage/high_availability/ob_transfer_handler.cpp index 6fe6d1624a..58947320d4 100644 --- a/src/storage/high_availability/ob_transfer_handler.cpp +++ b/src/storage/high_availability/ob_transfer_handler.cpp @@ -31,6 +31,7 @@ #include "share/ob_storage_ha_diagnose_struct.h" #include "storage/high_availability/ob_storage_ha_diagnose_mgr.h" #include "storage/tx/wrs/ob_weak_read_util.h" +#include "rootserver/mview/ob_collect_mv_merge_info_task.h" using namespace oceanbase::transaction; using namespace oceanbase::share; @@ -916,6 +917,37 @@ int ObTransferHandler::get_ls_active_trans_count_( return ret; } +int ObTransferHandler::get_dest_ls_mv_merge_scn_( + const share::ObTransferTaskInfo &task_info, + share::SCN &new_mv_merge_scn) +{ + int ret = OB_SUCCESS; + new_mv_merge_scn.set_min(); + ObMajorMVMergeInfo mv_merge_info; + uint64_t data_version = 0; + ObAddr dest_ls_leader; + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("transfer handler do not init", K(ret)); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(MTL_ID(), data_version))) { + LOG_WARN("fail to get data version", KR(ret)); + } else if (OB_UNLIKELY(data_version < DATA_VERSION_4_3_4_0)) { + } else if (!task_info.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("check start status src ls info get invalid argument", K(ret), K(task_info)); + } else if (OB_FAIL(get_ls_leader_(task_info.dest_ls_id_, dest_ls_leader))) { + LOG_WARN("failed to get dest ls leader", K(ret), K(task_info)); + } else if (OB_FAIL(rootserver::ObCollectMvMergeInfoTask::sync_get_ls_member_merge_info( + dest_ls_leader, task_info.tenant_id_, task_info.dest_ls_id_, + mv_merge_info, obrpc::ObRpcProxy::MAX_RPC_TIMEOUT, + true /*need check leader*/))) { + LOG_WARN("failed to get ls member merge info", K(ret), + K(dest_ls_leader), K(task_info), K(new_mv_merge_scn)); + } else if (OB_FALSE_IT(new_mv_merge_scn = mv_merge_info.major_mv_merge_scn_publish_)) { + } + return ret; +} + int ObTransferHandler::check_start_status_transfer_tablets_( const share::ObTransferTaskInfo &task_info) { @@ -923,6 +955,7 @@ int ObTransferHandler::check_start_status_transfer_tablets_( common::ObMemberList member_list; ObArray member_addr_list; const int64_t cluster_id = GCONF.cluster_id; + SCN new_mv_merge_scn; if (!is_inited_) { ret = OB_NOT_INIT; @@ -934,6 +967,8 @@ int ObTransferHandler::check_start_status_transfer_tablets_( LOG_WARN("failed to get src ls member list", K(ret), K(task_info)); } else if (OB_FAIL(member_list.get_addr_array(member_addr_list))) { LOG_WARN("failed to get addr array", K(ret), K(task_info), K(member_list)); + } else if (OB_FAIL(get_dest_ls_mv_merge_scn_(task_info, new_mv_merge_scn))) { + LOG_WARN("failed to get dest ls mv merge scn", K(ret), K(task_info)); } else { storage::ObCheckStartTransferTabletsProxy batch_proxy( *(GCTX.storage_rpc_proxy_), &obrpc::ObStorageRpcProxy::check_start_transfer_tablets); @@ -944,6 +979,7 @@ int ObTransferHandler::check_start_status_transfer_tablets_( arg.src_ls_id_ = task_info.src_ls_id_; arg.dest_ls_id_ = task_info.dest_ls_id_; arg.data_version_ = task_info.data_version_; + arg.new_mv_merge_scn_ = new_mv_merge_scn; const int64_t timeout = GCONF.rpc_timeout; const int64_t cluster_id = GCONF.cluster_id; const uint64_t group_id = share::OBCG_STORAGE; diff --git a/src/storage/high_availability/ob_transfer_handler.h b/src/storage/high_availability/ob_transfer_handler.h index 24cab480b8..8602bd50a4 100644 --- a/src/storage/high_availability/ob_transfer_handler.h +++ b/src/storage/high_availability/ob_transfer_handler.h @@ -161,6 +161,9 @@ private: int64_t &active_trans_count); int check_start_status_transfer_tablets_( const share::ObTransferTaskInfo &task_info); + int get_dest_ls_mv_merge_scn_( + const share::ObTransferTaskInfo &task_info, + share::SCN &new_mv_merge_scn); int get_ls_leader_( const share::ObLSID &ls_id, common::ObAddr &addr); diff --git a/src/storage/ls/ob_ls.cpp b/src/storage/ls/ob_ls.cpp index ac01fe5102..e77c9962df 100755 --- a/src/storage/ls/ob_ls.cpp +++ b/src/storage/ls/ob_ls.cpp @@ -133,6 +133,7 @@ int ObLS::init(const share::ObLSID &ls_id, const ObMigrationStatus &migration_status, const ObLSRestoreStatus &restore_status, const SCN &create_scn, + const ObMajorMVMergeInfo &major_mv_merge_info, const ObLSStoreFormat &store_format, observer::ObIMetaReport *reporter) { @@ -161,8 +162,9 @@ int ObLS::init(const share::ObLSID &ls_id, migration_status, restore_status, create_scn, + major_mv_merge_info, store_format))) { - LOG_WARN("failed to init ls meta", K(ret), K(tenant_id), K(ls_id)); + LOG_WARN("failed to init ls meta", K(ret), K(tenant_id), K(ls_id), K(major_mv_merge_info)); } else { rs_reporter_ = reporter; ls_freezer_.init(this); @@ -1543,6 +1545,9 @@ int ObLS::get_ls_info(ObLSVTInfo &ls_info) ls_info.tablet_change_checkpoint_scn_ = ls_meta_.get_tablet_change_checkpoint_scn(); ls_info.transfer_scn_ = ls_meta_.get_transfer_scn(); ls_info.tx_blocked_ = tx_blocked; + ls_info.mv_major_merge_scn_ = ls_meta_.get_major_mv_merge_info().major_mv_merge_scn_; + ls_info.mv_publish_scn_ = ls_meta_.get_major_mv_merge_info().major_mv_merge_scn_publish_; + ls_info.mv_safe_scn_ = ls_meta_.get_major_mv_merge_info().major_mv_merge_scn_safe_calc_; ls_info.required_data_disk_size_ = required_data_disk_size; if (tx_blocked) { TRANS_LOG(INFO, "current ls is blocked", K(ls_info)); diff --git a/src/storage/ls/ob_ls.h b/src/storage/ls/ob_ls.h index 0ccfcc9ab1..0c118be01c 100644 --- a/src/storage/ls/ob_ls.h +++ b/src/storage/ls/ob_ls.h @@ -67,13 +67,13 @@ #include "storage/high_availability/ob_ls_transfer_info.h" #include "observer/table/ttl/ob_tenant_tablet_ttl_mgr.h" #include "storage/ls/ob_ls_transfer_status.h" +#include "storage/mview/ob_major_mv_merge_info.h" #include "storage/ls/ob_freezer_define.h" #ifdef OB_BUILD_SHARED_STORAGE #include "storage/shared_storage/ob_private_block_gc_task.h" #include "storage/shared_storage/prewarm/ob_ls_prewarm_handler.h" #endif - namespace oceanbase { namespace observer @@ -107,6 +107,9 @@ struct ObLSVTInfo share::SCN tablet_change_checkpoint_scn_; share::SCN transfer_scn_; bool tx_blocked_; + share::SCN mv_major_merge_scn_; + share::SCN mv_publish_scn_; + share::SCN mv_safe_scn_; int64_t required_data_disk_size_; TO_STRING_KV(K_(ls_id), K_(replica_type), @@ -120,6 +123,9 @@ struct ObLSVTInfo K_(tablet_change_checkpoint_scn), K_(transfer_scn), K_(tx_blocked), + K_(mv_major_merge_scn), + K_(mv_publish_scn), + K_(mv_safe_scn), K_(required_data_disk_size)); }; @@ -241,6 +247,7 @@ public: const ObMigrationStatus &migration_status, const share::ObLSRestoreStatus &restore_status, const share::SCN &create_scn, + const ObMajorMVMergeInfo &major_mv_merge_info, const ObLSStoreFormat &store_format, observer::ObIMetaReport *reporter); // I am ready to work now. @@ -480,11 +487,14 @@ public: } CONST_DELEGATE_WITH_RET(ls_meta_, get_rebuild_seq, int64_t); CONST_DELEGATE_WITH_RET(ls_meta_, get_tablet_change_checkpoint_scn, share::SCN); - + DELEGATE_WITH_RET(ls_meta_, set_tablet_change_checkpoint_scn, int); int set_tablet_change_checkpoint_scn(const share::SCN &tablet_change_checkpoint_scn) { return ls_meta_.set_tablet_change_checkpoint_scn(ls_epoch_, tablet_change_checkpoint_scn); } + int set_major_mv_merge_scn(const share::SCN &scn) { return ls_meta_.set_major_mv_merge_scn(ls_epoch_, scn); } + int set_major_mv_merge_scn_safe_calc(const share::SCN &scn) { return ls_meta_.set_major_mv_merge_scn_safe_calc(ls_epoch_, scn); } + int set_major_mv_merge_scn_publish(const share::SCN &scn) { return ls_meta_.set_major_mv_merge_scn_publish(ls_epoch_, scn); } int set_restore_status( const share::ObLSRestoreStatus &restore_status, const int64_t rebuild_seq); diff --git a/src/storage/ls/ob_ls_meta.cpp b/src/storage/ls/ob_ls_meta.cpp index c23b8fdf57..c5d088c314 100644 --- a/src/storage/ls/ob_ls_meta.cpp +++ b/src/storage/ls/ob_ls_meta.cpp @@ -244,6 +244,90 @@ int ObLSMeta::set_tablet_change_checkpoint_scn( return ret; } +ObMajorMVMergeInfo ObLSMeta::get_major_mv_merge_info() const +{ + ObReentrantRLockGuard guard(rw_lock_); + return major_mv_merge_info_; +} + +int ObLSMeta::set_major_mv_merge_scn(const int64_t ls_epoch, const SCN &major_mv_merge_scn) +{ + int ret = OB_SUCCESS; + ObReentrantWLockGuard update_guard(update_lock_); + if (OB_FAIL(check_can_update_())) { + LOG_WARN("ls meta cannot update", K(ret), K(*this)); + } else if (major_mv_merge_info_.major_mv_merge_scn_ >= major_mv_merge_scn) { + LOG_INFO("old_scn is less than new_scn, skip", K(tenant_id_), K(ls_id_), + "old_scn", major_mv_merge_info_.major_mv_merge_scn_, "new_scn", major_mv_merge_scn); + } else { + ObLSMeta tmp(*this); + tmp.major_mv_merge_info_.major_mv_merge_scn_ = major_mv_merge_scn; + + if (OB_FAIL(write_slog_(ls_epoch, tmp))) { + LOG_WARN("write slog failed", K(ret)); + } else { + ObReentrantWLockGuard guard(rw_lock_); + LOG_INFO("update major_mv_merge_scn", K(tenant_id_), K(ls_id_), + "old_scn", major_mv_merge_info_.major_mv_merge_scn_, "new_scn", major_mv_merge_scn); + major_mv_merge_info_.major_mv_merge_scn_ = major_mv_merge_scn; + } + } + + return ret; +} + +int ObLSMeta::set_major_mv_merge_scn_safe_calc(const int64_t ls_epoch, const SCN &major_mv_merge_scn_safe_calc) +{ + int ret = OB_SUCCESS; + ObReentrantWLockGuard update_guard(update_lock_); + if (OB_FAIL(check_can_update_())) { + LOG_WARN("ls meta cannot update", K(ret), K(*this)); + } else if (major_mv_merge_info_.major_mv_merge_scn_safe_calc_ >= major_mv_merge_scn_safe_calc) { + LOG_INFO("old_scn is not less than new_scn, skip", K(tenant_id_), K(ls_id_), + "old_scn", major_mv_merge_info_.major_mv_merge_scn_safe_calc_, "new_scn", major_mv_merge_scn_safe_calc); + } else { + ObLSMeta tmp(*this); + tmp.major_mv_merge_info_.major_mv_merge_scn_safe_calc_ = major_mv_merge_scn_safe_calc; + + if (OB_FAIL(write_slog_(ls_epoch, tmp))) { + LOG_WARN("write slog failed", K(ret)); + } else { + ObReentrantWLockGuard guard(rw_lock_); + LOG_INFO("update major_mv_merge_scn_safe_calc", K(tenant_id_), K(ls_id_), + "old_scn", major_mv_merge_info_.major_mv_merge_scn_safe_calc_, "new_scn", major_mv_merge_scn_safe_calc); + major_mv_merge_info_.major_mv_merge_scn_safe_calc_ = major_mv_merge_scn_safe_calc; + } + } + + return ret; +} + +int ObLSMeta::set_major_mv_merge_scn_publish(const int64_t ls_epoch, const SCN &major_mv_merge_scn_publish) +{ + int ret = OB_SUCCESS; + ObReentrantWLockGuard update_guard(update_lock_); + if (OB_FAIL(check_can_update_())) { + LOG_WARN("ls meta cannot update", K(ret), K(*this)); + } else if (major_mv_merge_info_.major_mv_merge_scn_publish_ >= major_mv_merge_scn_publish) { + LOG_INFO("old_scn is not less than new_scn, skip", K(tenant_id_), K(ls_id_), + "old_scn", major_mv_merge_info_.major_mv_merge_scn_publish_, "new_scn", major_mv_merge_scn_publish); + } else { + ObLSMeta tmp(*this); + tmp.major_mv_merge_info_.major_mv_merge_scn_publish_ = major_mv_merge_scn_publish; + + if (OB_FAIL(write_slog_(ls_epoch, tmp))) { + LOG_WARN("write slog failed", K(ret)); + } else { + ObReentrantWLockGuard guard(rw_lock_); + LOG_INFO("update major_mv_merge_scn_publish", K(tenant_id_), K(ls_id_), + "old_scn", major_mv_merge_info_.major_mv_merge_scn_publish_, "new_scn", major_mv_merge_scn_publish); + major_mv_merge_info_.major_mv_merge_scn_publish_ = major_mv_merge_scn_publish; + } + } + + return ret; +} + share::SCN ObLSMeta::get_transfer_scn() const { ObReentrantRLockGuard guard(rw_lock_); @@ -703,12 +787,14 @@ int ObLSMeta::init( const ObMigrationStatus &migration_status, const share::ObLSRestoreStatus &restore_status, const SCN &create_scn, + const ObMajorMVMergeInfo &major_mv_merge_info, const ObLSStoreFormat &store_format) { int ret = OB_SUCCESS; if (OB_INVALID_ID == tenant_id || !ls_id.is_valid() || !ObMigrationStatusHelper::is_valid(migration_status) - || !restore_status.is_valid()) { + || !restore_status.is_valid() + || !major_mv_merge_info.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("init ls meta get invalid argument", K(ret), K(tenant_id), K(ls_id), K(migration_status), K(restore_status)); @@ -723,6 +809,7 @@ int ObLSMeta::init( gc_state_ = LSGCState::NORMAL; restore_status_ = restore_status; transfer_scn_ = SCN::min_scn(); + major_mv_merge_info_ = major_mv_merge_info; store_format_ = store_format; } return ret; diff --git a/src/storage/ls/ob_ls_meta.h b/src/storage/ls/ob_ls_meta.h index bfd4bd8e60..cf50b1a05a 100644 --- a/src/storage/ls/ob_ls_meta.h +++ b/src/storage/ls/ob_ls_meta.h @@ -117,6 +117,10 @@ public: int get_rebuild_info(ObLSRebuildInfo &rebuild_info) const; int get_create_type(int64_t &create_type) const; int check_ls_need_online(bool &need_online) const; + ObMajorMVMergeInfo get_major_mv_merge_info() const; + int set_major_mv_merge_scn(const int64_t ls_epoch, const SCN &major_mv_merge_scn); + int set_major_mv_merge_scn_safe_calc(const int64_t ls_epoch, const SCN &major_mv_merge_scn_safe_calc); + int set_major_mv_merge_scn_publish(const int64_t ls_epoch, const SCN &major_mv_merge_scn_publish); ObLSStoreFormat get_store_format() const; int init( @@ -125,6 +129,7 @@ public: const ObMigrationStatus &migration_status, const share::ObLSRestoreStatus &restore_status, const share::SCN &create_scn, + const ObMajorMVMergeInfo &major_mv_merge_info, const ObLSStoreFormat &store_format); ObReplicaType get_replica_type() const diff --git a/src/storage/multi_data_source/compile_utility/mds_register.h b/src/storage/multi_data_source/compile_utility/mds_register.h index 658e4a0b67..244a7b6154 100644 --- a/src/storage/multi_data_source/compile_utility/mds_register.h +++ b/src/storage/multi_data_source/compile_utility/mds_register.h @@ -74,6 +74,7 @@ #include "src/storage/multi_data_source/ob_start_transfer_in_mds_ctx.h" #include "src/storage/multi_data_source/ob_finish_transfer_in_mds_ctx.h" #include "src/share/ob_standby_upgrade.h" + #include "src/storage/mview/ob_major_mv_merge_info.h" #endif /**************************************************************************************************/ @@ -182,6 +183,14 @@ _GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION_(HELPER_CLASS, BUFFER_CTX_TYPE, ID, ENU ::oceanbase::storage::mds::MdsCtx,\ 34,\ TABLET_BINDING) + GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION(::oceanbase::storage::ObMVPublishSCNHelper,\ + ::oceanbase::storage::ObUnUseCtx, \ + 35,\ + MV_PUBLISH_SCN) + GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION(::oceanbase::storage::ObMVNoticeSafeHelper,\ + ::oceanbase::storage::ObUnUseCtx, \ + 36,\ + MV_NOTICE_SAFE) // # 余留位置（此行之前占位） #undef GENERATE_MDS_FRAME_CODE_FOR_TRANSACTION #endif diff --git a/src/storage/mview/ob_major_mv_merge_info.cpp b/src/storage/mview/ob_major_mv_merge_info.cpp index daf472f5f8..7e9f14812a 100644 --- a/src/storage/mview/ob_major_mv_merge_info.cpp +++ b/src/storage/mview/ob_major_mv_merge_info.cpp @@ -14,6 +14,10 @@ #include "ob_major_mv_merge_info.h" #include "share/ob_table_range.h" +#include "storage/tablet/ob_tablet_iterator.h" +#include "storage/tx_storage/ob_ls_service.h" +#include "storage/tablet/ob_tablet.h" +#include "rootserver/mview/ob_collect_mv_merge_info_task.h" namespace oceanbase { @@ -39,10 +43,313 @@ bool ObMajorMVMergeInfo::is_valid() const { return major_mv_merge_scn_.is_valid() && major_mv_merge_scn_safe_calc_.is_valid() - && major_mv_merge_scn_publish_.is_valid(); + && major_mv_merge_scn_publish_.is_valid() + && major_mv_merge_scn_ <= major_mv_merge_scn_safe_calc_ + && major_mv_merge_scn_safe_calc_ <= major_mv_merge_scn_publish_; } +void ObMajorMVMergeInfo::operator=(const ObMajorMVMergeInfo &other) +{ + major_mv_merge_scn_ = other.major_mv_merge_scn_; + major_mv_merge_scn_safe_calc_ = other.major_mv_merge_scn_safe_calc_; + major_mv_merge_scn_publish_ = other.major_mv_merge_scn_publish_; +} + +int ObMajorMVMergeInfo::init(const share::SCN &major_mv_merge_scn, + const share::SCN &major_mv_merge_scn_safe_calc, + const share::SCN &major_mv_merge_scn_publish) +{ + int ret = OB_SUCCESS; + + if (!major_mv_merge_scn.is_valid() + || !major_mv_merge_scn_safe_calc.is_valid() + || !major_mv_merge_scn_publish.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(major_mv_merge_scn), K(major_mv_merge_scn_safe_calc), K(major_mv_merge_scn_publish)); + } else { + major_mv_merge_scn_ = major_mv_merge_scn; + major_mv_merge_scn_safe_calc_ = major_mv_merge_scn_safe_calc; + major_mv_merge_scn_publish_ = major_mv_merge_scn_publish; + } + + return ret; +} OB_SERIALIZE_MEMBER(ObMajorMVMergeInfo, major_mv_merge_scn_, major_mv_merge_scn_safe_calc_, major_mv_merge_scn_publish_); +ObUpdateMergeScnArg::ObUpdateMergeScnArg() + : ls_id_(), + merge_scn_(share::ObScnRange::MIN_SCN) +{ +} + +void ObUpdateMergeScnArg::reset() +{ + merge_scn_ = share::ObScnRange::MIN_SCN; + ls_id_.reset(); +} + +bool ObUpdateMergeScnArg::is_valid() const +{ + return merge_scn_.is_valid() && ls_id_.is_valid(); +} + +OB_SERIALIZE_MEMBER(ObUpdateMergeScnArg, ls_id_, merge_scn_); + +#define SET_MERGE_SCN(set_func) \ + ObUpdateMergeScnArg arg; \ + ObLSHandle ls_handle; \ + ObLS *ls = NULL; \ + int64_t pos = 0; \ + if (OB_FAIL(ret)) { \ + } else if (OB_ISNULL(buf) || OB_UNLIKELY(len <= 0)) { \ + ret = OB_INVALID_ARGUMENT; \ + LOG_WARN("invalid args", KR(ret), KP(buf), K(len)); \ + } else if (OB_FAIL(arg.deserialize(buf, len, pos))) { \ + LOG_WARN("failed to deserialize", KR(ret)); \ + } else if (OB_UNLIKELY(!arg.is_valid())) { \ + ret = OB_ERR_UNEXPECTED; \ + LOG_WARN("arg is invalid", KR(ret), K(arg)); \ + } else if (OB_FAIL(MTL(ObLSService*)->get_ls(arg.ls_id_, ls_handle, ObLSGetMod::TABLET_MOD))) { \ + LOG_WARN("failed to get ls", KR(ret), K(arg)); \ + } else if (OB_UNLIKELY(NULL == (ls = ls_handle.get_ls()))) { \ + ret = OB_ERR_UNEXPECTED; \ + LOG_WARN("ls should not be NULL", KR(ret), K(arg), KPC(ls)); \ + } else if (OB_FAIL(ls->set_func(arg.merge_scn_))) { \ + LOG_WARN("failed to "#set_func, KR(ret), K(arg), KPC(ls)); \ + } \ + LOG_INFO(#set_func" finish", KR(ret), K(arg)); + + +int ObMVPublishSCNHelper::on_register( + const char *buf, + const int64_t len, + mds::BufferCtx &ctx) +{ + int ret = OB_SUCCESS; + SET_MERGE_SCN(set_major_mv_merge_scn_publish); + return ret; +} + + +int ObMVPublishSCNHelper::on_replay( + const char *buf, + const int64_t len, + const share::SCN scn, + mds::BufferCtx &ctx) +{ + int ret = OB_SUCCESS; + share::ObTenantRole::Role tenant_role = MTL_GET_TENANT_ROLE_CACHE(); + if (is_invalid_tenant(tenant_role)) { + ret = OB_EAGAIN; + LOG_WARN("tenant role cache is invalid", KR(ret)); + } else if (is_standby_tenant(tenant_role)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("new mview are temporarily not support in standy tenant", KR(ret)); + } + SET_MERGE_SCN(set_major_mv_merge_scn_publish); + return ret; +} + +int ObMVNoticeSafeHelper::on_register( + const char *buf, + const int64_t len, + mds::BufferCtx &ctx) +{ + int ret = OB_SUCCESS; + SET_MERGE_SCN(set_major_mv_merge_scn_safe_calc); + return ret; +} + +int ObMVNoticeSafeHelper::on_replay( + const char *buf, + const int64_t len, + const share::SCN scn, + mds::BufferCtx &ctx) +{ + int ret = OB_SUCCESS; + share::ObTenantRole::Role tenant_role = MTL_GET_TENANT_ROLE_CACHE(); + if (is_invalid_tenant(tenant_role)) { + ret = OB_EAGAIN; + LOG_WARN("tenant role cache is invalid", KR(ret)); + } else if (is_standby_tenant(tenant_role)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("new mview are temporarily not support in standy tenant", KR(ret)); + } + SET_MERGE_SCN(set_major_mv_merge_scn_safe_calc); + return ret; +} + +int ObMVCheckReplicaHelper::get_and_update_merge_info( + const share::ObLSID &ls_id, + ObMajorMVMergeInfo &info) +{ + int ret = OB_SUCCESS; + ObLSHandle ls_handle; + ObLS *ls = NULL; + ObLSTabletIterator tablet_iter(ObMDSGetTabletMode::READ_WITHOUT_CHECK); + bool skip_update = false; + if (!ls_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("arg is not valid", KR(ret), K(ls_id)); + } else if (OB_FAIL(MTL(ObLSService*)->get_ls(ls_id, ls_handle, ObLSGetMod::TABLET_MOD))) { + LOG_WARN("failed to get ls", KR(ret), K(ls_id)); + } else if (OB_UNLIKELY(NULL == (ls = ls_handle.get_ls()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls should not be NULL", KR(ret), KPC(ls)); + } else if (FALSE_IT(info = ls->get_ls_meta().get_major_mv_merge_info())) { + } else if (!info.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("info is invalid", KR(ret), K(info)); + } else if (!info.need_update_major_mv_merge_scn()) { + STORAGE_LOG(INFO, "no need update", KR(ret), K(info), KPC(ls)); + } else if (OB_FAIL(ls->get_tablet_svr()->build_tablet_iter(tablet_iter))) { + STORAGE_LOG(WARN, "failed to build ls tablet iter", KR(ret), KPC(ls)); + } else { + share::SCN min_major_mv_merge_scn; + min_major_mv_merge_scn.set_max(); + while (OB_SUCC(ret) && !skip_update) { + ObTabletHandle tablet_handle; + ObTablet *tablet = NULL; + ObStorageSchema *storage_schema = nullptr; + ObArenaAllocator allocator; + if (OB_FAIL(tablet_iter.get_next_tablet(tablet_handle))) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + break; + } else { + STORAGE_LOG(WARN, "failed to get tablet", KR(ret), K(tablet_handle)); + } + } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "invalid tablet handle", KR(ret), K(tablet_handle)); + } else if (OB_ISNULL(tablet = tablet_handle.get_obj())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "tablet is NULL", KR(ret)); + } else if (tablet->is_ls_inner_tablet()) { + // skip ls inner tablet + } else if (OB_FAIL(tablet->load_storage_schema(allocator, storage_schema))) { + LOG_WARN("load storage schema failed", K(ret), K(ls_id), KPC(tablet)); + } else if (OB_ISNULL(storage_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("storage schema is NULL", K(ret), K(ls_id), KPC(tablet)); + } else if (storage_schema->is_mv_major_refresh()) { + const int64_t snapshot = tablet->get_last_major_snapshot_version(); + if (0 == snapshot || 1 == snapshot) { + skip_update = true; + LOG_INFO("snapshot is invalid, skip update", K(ret), K(info), K(snapshot), K(ls_id), KPC(tablet)); + } else if (min_major_mv_merge_scn.get_val_for_gts() > snapshot + && OB_FAIL(min_major_mv_merge_scn.convert_for_gts(snapshot))) { + LOG_WARN("failed to convert_for_gts", K(ret), K(info), K(snapshot), KPC(ls)); + } + } + } + if (OB_FAIL(ret)) { + } else if (skip_update) { + } else if (min_major_mv_merge_scn < info.major_mv_merge_scn_) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("min_major_mv_merge_scn is less info.major_mv_merge_scn", K(ret), K(info), K(min_major_mv_merge_scn)); + } + // there is no new mv tablet + else if (min_major_mv_merge_scn.is_max()) { + if (OB_FAIL(ls->set_major_mv_merge_scn(info.major_mv_merge_scn_safe_calc_))) { + LOG_WARN("failed to set_major_mv_merge_scn", K(ret), K(info), K(min_major_mv_merge_scn), KPC(ls)); + } else { + info.major_mv_merge_scn_ = info.major_mv_merge_scn_safe_calc_; + } + } else if (min_major_mv_merge_scn >= info.major_mv_merge_scn_safe_calc_) { + if (OB_FAIL(ls->set_major_mv_merge_scn(info.major_mv_merge_scn_safe_calc_))) { + LOG_WARN("failed to set_major_mv_merge_scn", K(ret), K(info), K(min_major_mv_merge_scn), KPC(ls)); + } else { + info.major_mv_merge_scn_ = info.major_mv_merge_scn_safe_calc_; + } + } + } + return ret; +} + +int ObMVCheckReplicaHelper::get_merge_info( + const share::ObLSID &ls_id, + ObMajorMVMergeInfo &info) +{ + int ret = OB_SUCCESS; + ObLSHandle ls_handle; + ObLS *ls = NULL; + if (!ls_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("arg is not valid", KR(ret), K(ls_id)); + } else if (OB_FAIL(MTL(ObLSService*)->get_ls(ls_id, ls_handle, ObLSGetMod::TABLET_MOD))) { + LOG_WARN("failed to get ls", KR(ret), K(ls_id)); + } else if (OB_UNLIKELY(NULL == (ls = ls_handle.get_ls()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls should not be NULL", KR(ret), KPC(ls)); + } else if (FALSE_IT(info = ls->get_ls_meta().get_major_mv_merge_info())) { + } else if (!info.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("info is invalid", KR(ret), K(info)); + } + return ret; +} + +ERRSIM_POINT_DEF(ERRSIM_RECONFIG_CHECK_FAILED); + +int ObMVCheckReplicaHelper::check_can_add_member( + const common::ObAddr &server, + const uint64_t tenant_id, + const share::ObLSID &ls_id, + const uint64_t rpc_timeout) +{ + int ret = OB_SUCCESS; + + ObMajorMVMergeInfo leader_merge_info; + ObMajorMVMergeInfo member_merge_info; + if (!server.is_valid() || tenant_id == OB_INVALID_TENANT_ID || !ls_id.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), K(ls_id), K(tenant_id), K(server)); + } else if (ls_id.is_sys_ls()) { + // do nothing + } else if (OB_FAIL(rootserver::ObCollectMvMergeInfoTask::sync_get_ls_member_merge_info(server, + tenant_id, + ls_id, + member_merge_info, + rpc_timeout, + false/*check_leader*/))) { + LOG_WARN("sync get ls member merge info failed", K(ret), + K(server), K(ls_id), K(tenant_id)); + } else if (OB_FAIL(get_merge_info(ls_id, leader_merge_info))) { + LOG_WARN("get and update local merge info failed", K(ret), K(ls_id)); + } else if (leader_merge_info.major_mv_merge_scn_ > member_merge_info.major_mv_merge_scn_) { + ret = NEW_MV_MAJOR_VERSION_NOT_MATCH; + LOG_WARN("ls mv merge scn is small than leader", K(ret), + K(leader_merge_info), K(ls_id), K(member_merge_info)); + } else { + LOG_INFO("ls reconfig check success", K(ret), + K(leader_merge_info), K(ls_id), K(member_merge_info)); + } + if (NEW_MV_MAJOR_VERSION_NOT_MATCH == ret) { + if (OB_FAIL(rootserver::ObCollectMvMergeInfoTask::sync_get_ls_member_merge_info(server, + tenant_id, + ls_id, + member_merge_info, + rpc_timeout, + false/*check_leader*/, + true/*need_update*/))) { + LOG_WARN("sync get ls member merge info failed", K(ret), + K(server), K(ls_id), K(tenant_id)); + } else if (leader_merge_info.major_mv_merge_scn_ > member_merge_info.major_mv_merge_scn_) { + ret = NEW_MV_MAJOR_VERSION_NOT_MATCH; + LOG_WARN("ls mv merge scn is small than leader", K(ret), + K(leader_merge_info), K(ls_id), K(member_merge_info)); + } + LOG_INFO("push dest mv merge scn to avoid failed", K(ret), K(tenant_id), K(server), + K(ls_id), K(member_merge_info), K(leader_merge_info)); + } + if (OB_UNLIKELY(ERRSIM_RECONFIG_CHECK_FAILED == NEW_MV_MAJOR_VERSION_NOT_MATCH) && !ls_id.is_sys_ls()) { + ret = NEW_MV_MAJOR_VERSION_NOT_MATCH; + LOG_INFO("error sim to check failed", K(ret), + K(leader_merge_info), K(ls_id), K(member_merge_info)); + } + return ret; +} + } } diff --git a/src/storage/mview/ob_major_mv_merge_info.h b/src/storage/mview/ob_major_mv_merge_info.h index c972f711f2..79038b41f6 100644 --- a/src/storage/mview/ob_major_mv_merge_info.h +++ b/src/storage/mview/ob_major_mv_merge_info.h @@ -14,6 +14,7 @@ #define OCEABASE_MAJOR_MV_MERGE_INFO_ #include "share/scn.h" +#include "storage/multi_data_source/buffer_ctx.h" namespace oceanbase { @@ -28,6 +29,12 @@ public: ~ObMajorMVMergeInfo() = default; bool is_valid() const; void reset(); + bool need_update_major_mv_merge_scn() + { return major_mv_merge_scn_ < major_mv_merge_scn_safe_calc_; } + void operator=(const ObMajorMVMergeInfo &other); + int init(const share::SCN &major_mv_merge_scn, + const share::SCN &major_mv_merge_scn_safe_calc, + const share::SCN &major_mv_merge_scn_publish); TO_STRING_KV(K_(major_mv_merge_scn), K_(major_mv_merge_scn_safe_calc), K_(major_mv_merge_scn_publish)); @@ -36,6 +43,72 @@ public: share::SCN major_mv_merge_scn_publish_; }; +struct ObUpdateMergeScnArg final +{ + OB_UNIS_VERSION(1); +public: + ObUpdateMergeScnArg(); + ~ObUpdateMergeScnArg() = default; + bool is_valid() const; + void reset(); + + TO_STRING_KV(K_(ls_id), K_(merge_scn)); + + share::ObLSID ls_id_; + share::SCN merge_scn_; +}; + +struct ObMVPublishSCNHelper +{ + static int on_register( + const char *buf, + const int64_t len, + mds::BufferCtx &ctx); + + static int on_replay( + const char *buf, + const int64_t len, + const share::SCN scn, + mds::BufferCtx &ctx); +}; + +struct ObMVNoticeSafeHelper +{ + static int on_register( + const char *buf, + const int64_t len, + mds::BufferCtx &ctx); + + static int on_replay( + const char *buf, + const int64_t len, + const share::SCN scn, + mds::BufferCtx &ctx); +}; + +struct ObMVCheckReplicaHelper +{ + static int get_and_update_merge_info( + const share::ObLSID &ls_id, + ObMajorMVMergeInfo &info); + static int check_can_add_member( + const common::ObAddr &server, + const uint64_t tenant_id, + const share::ObLSID &ls_id, + const uint64_t rpc_timeout); + static int get_merge_info( + const share::ObLSID &ls_id, + ObMajorMVMergeInfo &info); +}; + +struct ObUnUseCtx : public mds::BufferCtx +{ + TO_STRING_EMPTY(); + int serialize(char *buf, const int64_t len, int64_t &pos) const { return OB_SUCCESS; } + int deserialize(const char *buf, const int64_t len, int64_t &pos) { return OB_SUCCESS; } + int64_t get_serialize_size() const { return 0; } + const mds::MdsWriter get_writer() const { return mds::MdsWriter(mds::WriterType::TRANSACTION, 0); } +}; } } diff --git a/src/storage/mview/ob_mview_sched_job_utils.cpp b/src/storage/mview/ob_mview_sched_job_utils.cpp index 77c3d276aa..0c1a999e77 100644 --- a/src/storage/mview/ob_mview_sched_job_utils.cpp +++ b/src/storage/mview/ob_mview_sched_job_utils.cpp @@ -23,7 +23,9 @@ #include "lib/string/ob_string.h" #include "lib/mysqlclient/ob_isql_client.h" #include "share/ob_time_utility2.h" +#include "share/ob_global_stat_proxy.h" #include "common/object/ob_object.h" +#include "share/backup/ob_backup_data_table_operator.h" #include "share/schema/ob_schema_struct.h" #include "share/schema/ob_mview_info.h" #include "share/schema/ob_mlog_info.h" @@ -183,10 +185,30 @@ int ObMViewSchedJobUtils::add_mview_info_and_refresh_job(ObISQLClient &sql_clien ObArenaAllocator allocator("CreateMVTmp"); SCN curr_ts; mview_info.reset(); + share::ObGlobalStatProxy stat_proxy(sql_client, tenant_id); + share::SCN major_refresh_mv_merge_scn; + ObArray backup_jobs; + uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id); if (refresh_info == nullptr) { ret = OB_INVALID_ARGUMENT; LOG_WARN("refresh_info is null", KR(ret)); - } else if (!refresh_info->start_time_.is_null() || !refresh_info->next_time_expr_.empty()) { + } else if (ObMVRefreshMode::MAJOR_COMPACTION == refresh_info->refresh_mode_) { + if (OB_FAIL(acquire_major_refresh_mv_merge_scn_(sql_client, tenant_id))) { + LOG_WARN("failed to acquire major refresh mv merge scn", KR(ret), K(tenant_id)); + } else if (OB_FAIL(share::ObBackupJobOperator::get_jobs( + *GCTX.sql_proxy_, meta_tenant_id, false /*select for update*/, backup_jobs))) { + LOG_WARN("failed to get backup jobs", KR(ret), K(tenant_id)); + } else if (!backup_jobs.empty()) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("[MAJ_REF_MV] backup jobs exist, can not create materialized view", K(ret), + K(tenant_id)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "backup jobs exist, can not create materialized view, please " + "try again after backup jobs are finished"); + } + } + + if (OB_SUCC(ret) && + (!refresh_info->start_time_.is_null() || !refresh_info->next_time_expr_.empty())) { ObString job_prefix(ObMViewInfo::MVIEW_REFRESH_JOB_PREFIX); int64_t job_id = OB_INVALID_ID; @@ -225,6 +247,7 @@ int ObMViewSchedJobUtils::add_mview_info_and_refresh_job(ObISQLClient &sql_clien } } } + if (OB_SUCC(ret)) { if (OB_FAIL(OB_TS_MGR.get_ts_sync(tenant_id, GCONF.rpc_timeout, @@ -243,7 +266,13 @@ int ObMViewSchedJobUtils::add_mview_info_and_refresh_job(ObISQLClient &sql_clien mview_info.set_refresh_mode(refresh_info->refresh_mode_); mview_info.set_refresh_method(refresh_info->refresh_method_); mview_info.set_refresh_job(refresh_job); - mview_info.set_last_refresh_scn(curr_ts.get_val_for_inner_table_field()); + // TODO: we should set last_refresh_scn to 0 for all kind of mview, and fix the mlog recycle + // problem later. + if (ObMVRefreshMode::MAJOR_COMPACTION == refresh_info->refresh_mode_) { + mview_info.set_last_refresh_scn(0); + } else { + mview_info.set_last_refresh_scn(curr_ts.get_val_for_inner_table_field()); + } mview_info.set_schema_version(schema_version); if (refresh_info->start_time_.is_timestamp()) { mview_info.set_refresh_start(refresh_info->start_time_.get_timestamp()); @@ -451,5 +480,60 @@ int ObMViewSchedJobUtils::resolve_date_expr_to_timestamp( return ret; } +int ObMViewSchedJobUtils::acquire_major_refresh_mv_merge_scn_(common::ObISQLClient &trans, + const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + share::SCN major_refresh_mv_merge_scn; + ObMySQLTransaction tmp_trans; + common::ObISQLClient *sql_proxy = GCTX.sql_proxy_; + + // firstly, check if major_refresh_mv_merge_scn has been set, if not, set it to 0 + if (OB_UNLIKELY(OB_ISNULL(sql_proxy))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else if (OB_FAIL(tmp_trans.start(sql_proxy, tenant_id))) { + LOG_WARN("fail to start trans", KR(ret), K(tenant_id)); + } else { + share::ObGlobalStatProxy tmp_proxy(tmp_trans, tenant_id); + if (OB_FAIL(tmp_proxy.get_major_refresh_mv_merge_scn(true /*select for update*/, + major_refresh_mv_merge_scn))) { + if (OB_ERR_NULL_VALUE == ret) { + ret = OB_SUCCESS; + LOG_INFO("[MAJ_REF_MV] major_refresh_mv_merge_scn has not been set"); + major_refresh_mv_merge_scn.set_min(); + if (OB_FAIL(tmp_proxy.update_major_refresh_mv_merge_scn(major_refresh_mv_merge_scn, + false /*is incremental*/))) { + LOG_WARN("fail to update major_refresh_mv_merge_scn", KR(ret), + K(major_refresh_mv_merge_scn)); + } else { + LOG_INFO("[MAJ_REF_MV] init major_refresh_mv_merge_scn", K(tenant_id), + K(major_refresh_mv_merge_scn)); + } + } else { + LOG_WARN("fail to get major_refresh_mv_merge_scn", KR(ret), K(tenant_id)); + } + } + } + + if (tmp_trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = tmp_trans.end(OB_SUCC(ret)))) { + LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + } + + if (OB_SUCC(ret)) { + share::ObGlobalStatProxy stat_proxy(trans, tenant_id); + if (OB_FAIL(stat_proxy.get_major_refresh_mv_merge_scn(true /*select for update*/, + major_refresh_mv_merge_scn))) { + LOG_WARN("fail to get major_refresh_mv_merge_scn", KR(ret), K(tenant_id)); + } + } + + return ret; +} + } // end of storage } // end of oceanbase diff --git a/src/storage/mview/ob_mview_sched_job_utils.h b/src/storage/mview/ob_mview_sched_job_utils.h index b62a5153ba..c82141e743 100644 --- a/src/storage/mview/ob_mview_sched_job_utils.h +++ b/src/storage/mview/ob_mview_sched_job_utils.h @@ -14,6 +14,7 @@ #include "lib/ob_define.h" #include "sql/parser/parse_node.h" +#include "share/scn.h" namespace oceanbase { @@ -101,6 +102,9 @@ public: const ParseNode &node, common::ObIAllocator &allocator, int64_t ×tamp); +private: + static int acquire_major_refresh_mv_merge_scn_(common::ObISQLClient &trans, + const uint64_t tenant_id); }; } // namespace storage } // namespace oceanbase diff --git a/src/storage/ob_storage_rpc.cpp b/src/storage/ob_storage_rpc.cpp index e5de65987d..7f22d189f3 100644 --- a/src/storage/ob_storage_rpc.cpp +++ b/src/storage/ob_storage_rpc.cpp @@ -913,7 +913,7 @@ ObTransferTabletInfoArg::ObTransferTabletInfoArg() dest_ls_id_(), tablet_list_(), data_version_(0), - new_mv_merge_scn_() + new_mv_merge_scn_(share::ObScnRange::MIN_SCN) { } @@ -924,7 +924,7 @@ void ObTransferTabletInfoArg::reset() dest_ls_id_.reset(); tablet_list_.reset(); data_version_ = 0; - new_mv_merge_scn_.reset(); + new_mv_merge_scn_ = share::ObScnRange::MIN_SCN; } int ObTransferTabletInfoArg::assign(const ObTransferTabletInfoArg &other) @@ -950,7 +950,8 @@ bool ObTransferTabletInfoArg::is_valid() const return OB_INVALID_ID != tenant_id_ && src_ls_id_.is_valid() && dest_ls_id_.is_valid() - && !tablet_list_.empty(); + && !tablet_list_.empty() + && new_mv_merge_scn_.is_valid(); } OB_SERIALIZE_MEMBER(ObTransferTabletInfoArg, tenant_id_, src_ls_id_, dest_ls_id_, tablet_list_, data_version_, new_mv_merge_scn_); @@ -2459,6 +2460,9 @@ int ObCheckStartTransferTabletsDelegate::process() LOG_WARN("failed to check start transfer out tablets", K(ret), K(arg_)); } else if (OB_FAIL(check_start_transfer_in_tablets_())) { LOG_WARN("failed to check start transfer in tablets", K(ret), K(arg_)); + } else if (share::ObScnRange::MIN_SCN != arg_.new_mv_merge_scn_ + && OB_FAIL(check_start_transfer_in_mv_tablets_())) { + LOG_WARN("failed to check start transfer in mv tablets", K(ret), K(arg_)); } } return ret; @@ -2555,6 +2559,56 @@ int ObCheckStartTransferTabletsDelegate::check_start_transfer_out_tablets_() return ret; } +int ObCheckStartTransferTabletsDelegate::check_start_transfer_in_mv_tablets_() +{ + int ret = OB_SUCCESS; + ObLSHandle ls_handle; + ObLSService *ls_service = nullptr; + ObLS *src_ls = nullptr; + ObStorageSchema *storage_schema = nullptr; + ObArenaAllocator allocator; + if (OB_ISNULL(ls_service = MTL(ObLSService *))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "ls service should not be null", K(ret), KP(ls_service)); + } else if (OB_FAIL(ls_service->get_ls(arg_.src_ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD))) { + LOG_WARN("failed to get src ls", K(ret), K(arg_)); + } else if (OB_ISNULL(src_ls = ls_handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("src ls should not be NULL", K(ret), K(arg_), KP(src_ls)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < arg_.tablet_list_.count(); ++i) { + const ObTransferTabletInfo &tablet_info = arg_.tablet_list_.at(i); + ObTabletHandle tablet_handle; + ObTablet *tablet = nullptr; + if (OB_FAIL(src_ls->get_tablet(tablet_info.tablet_id_, tablet_handle, 0, + ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { + LOG_WARN("failed to get tablet", K(ret), K(tablet_info)); + } else if (OB_ISNULL(tablet = tablet_handle.get_obj())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet should not be NULL", K(ret), KP(tablet)); + } else if (tablet->is_ls_inner_tablet()) { + // skip ls inner tablet + } else if (OB_FAIL(tablet->load_storage_schema(allocator, storage_schema))) { + LOG_WARN("load storage schema failed", K(ret), KPC(tablet)); + } else if (OB_ISNULL(storage_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("storage schema is NULL", K(ret), KPC(tablet)); + } else if (storage_schema->is_mv_major_refresh()) { + const int64_t snapshot = tablet->get_last_major_snapshot_version(); + if (0 == snapshot) { + LOG_INFO("check major_mv merge_scn snapshot is 0, there is no major sstable", K(ret), K(arg_), K(snapshot), KPC(tablet)); + } else if (arg_.new_mv_merge_scn_.get_val_for_gts() > snapshot) { + ret = NEW_MV_MAJOR_VERSION_NOT_MATCH; + LOG_WARN("new mv major version is not match", K(ret), K(arg_), K(snapshot), KPC(src_ls), KPC(tablet)); + } else { + LOG_INFO("check major_mv merge_scn success", K(src_ls->get_ls_id()), K(tablet->get_tablet_id()), K(snapshot), K(arg_)); + } + } + } + } + return ret; +} + int ObCheckStartTransferTabletsDelegate::check_start_transfer_in_tablets_() { int ret = OB_SUCCESS; diff --git a/src/storage/ob_storage_rpc.h b/src/storage/ob_storage_rpc.h index 7ea7988c8a..8ab4cf1d6a 100644 --- a/src/storage/ob_storage_rpc.h +++ b/src/storage/ob_storage_rpc.h @@ -557,7 +557,7 @@ public: share::ObLSID dest_ls_id_; common::ObSArray tablet_list_; uint64_t data_version_; - share::SCN new_mv_merge_scn_; // placeholder for feature collect_mv + share::SCN new_mv_merge_scn_; private: DISALLOW_COPY_AND_ASSIGN(ObTransferTabletInfoArg); }; @@ -1119,6 +1119,7 @@ public: private: int check_start_transfer_out_tablets_(); int check_start_transfer_in_tablets_(); + int check_start_transfer_in_mv_tablets_(); // Major sstable or ddl sstable needs to exist in src_tablet int check_transfer_out_tablet_sstable_(const ObTablet *tablet); diff --git a/src/storage/ob_storage_schema.cpp b/src/storage/ob_storage_schema.cpp index bdf914a7c6..a49f0f4b0f 100644 --- a/src/storage/ob_storage_schema.cpp +++ b/src/storage/ob_storage_schema.cpp @@ -420,6 +420,7 @@ ObStorageSchema::ObStorageSchema() skip_idx_attr_array_(), store_column_cnt_(0), has_all_column_group_(false), + mv_mode_(), is_inited_(false) { } @@ -445,6 +446,8 @@ int ObStorageSchema::init( } else if (OB_UNLIKELY(!input_schema.is_valid())) { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "invalid args", K(ret), K(input_schema), K(skip_column_info)); + } else if (OB_FAIL(copy_from(input_schema))) { + STORAGE_LOG(WARN, "failed to copy from table schema", K(ret), K(input_schema)); } else if (FALSE_IT(column_info_simplified_ = skip_column_info)) { } else if (OB_UNLIKELY(generate_cs_replica_cg_array && column_info_simplified_)) { ret = OB_INVALID_ARGUMENT; @@ -457,7 +460,6 @@ int ObStorageSchema::init( skip_idx_attr_array_.set_allocator(&allocator); storage_schema_version_ = compat_version; - copy_from(input_schema); compat_mode_ = static_cast(compat_mode); } @@ -504,6 +506,8 @@ int ObStorageSchema::init( } else if (OB_UNLIKELY(!old_schema.is_valid())) { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "invalid args", K(ret), K(old_schema), K(skip_column_info)); + } else if (OB_FAIL(copy_from(old_schema))) { + STORAGE_LOG(WARN, "failed to copy from old schema", K(ret), K(old_schema)); } else if (FALSE_IT(column_info_simplified_ = (skip_column_info || old_schema.column_info_simplified_))) { } else if (OB_UNLIKELY(generate_cs_replica_cg_array && (column_info_simplified_ || column_group_schema != nullptr))) { ret = OB_INVALID_ARGUMENT; @@ -516,7 +520,6 @@ int ObStorageSchema::init( skip_idx_attr_array_.set_allocator(&allocator); storage_schema_version_ = old_schema.storage_schema_version_; - copy_from(old_schema); compat_mode_ = old_schema.compat_mode_; is_cs_replica_compat_ = old_schema.is_cs_replica_compat_; compressor_type_ = old_schema.compressor_type_; @@ -645,6 +648,7 @@ void ObStorageSchema::reset() info_ = 0; table_type_ = MAX_TABLE_TYPE; table_mode_.reset(); + mv_mode_.reset(); index_type_ = INDEX_TYPE_IS_NOT; row_store_type_ = ObStoreFormat::get_default_row_store_type(); schema_version_ = OB_INVALID_VERSION; @@ -692,6 +696,7 @@ bool ObStorageSchema::is_valid() const || pctfree_ < 0 || table_type_ >= MAX_TABLE_TYPE || !table_mode_.is_valid() + || !mv_mode_.is_valid() || index_type_ >= INDEX_TYPE_MAX || !check_column_array_valid(rowkey_array_) || !check_column_array_valid(column_array_) @@ -734,7 +739,7 @@ int ObStorageSchema::serialize(char *buf, const int64_t buf_len, int64_t &pos) c ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "invalid args", K(ret), K(buf), K(buf_len), K(pos)); } else if (STORAGE_SCHEMA_VERSION <= storage_schema_version_ - && STORAGE_SCHEMA_VERSION_V3 >= storage_schema_version_) { + && STORAGE_SCHEMA_VERSION_LATEST >= storage_schema_version_) { LST_DO_CODE(OB_UNIS_ENCODE, storage_schema_version_, info_, @@ -769,6 +774,9 @@ int ObStorageSchema::serialize(char *buf, const int64_t buf_len, int64_t &pos) c STORAGE_LOG(WARN, "failed to serialize skip idx attr array", K_(skip_idx_attr_array)); } } + if (OB_SUCC(ret) && storage_schema_version_ >= STORAGE_SCHEMA_VERSION_V4) { + OB_UNIS_ENCODE(mv_mode_); + } } else { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "invalid storage schema version", K(ret), K_(storage_schema_version)); @@ -788,13 +796,10 @@ int ObStorageSchema::serialize_column_array(char *buf, const int64_t data_len, i if (OB_FAIL(column_array_.at(i).legacy_serialize(buf, data_len, pos))) { STORAGE_LOG(WARN, "Fail to serialize column schema for legacy version", K(ret), K(i), K_(column_array)); } - } else if (STORAGE_SCHEMA_VERSION_V3 == storage_schema_version_) { + } else if (STORAGE_SCHEMA_VERSION_V3 <= storage_schema_version_) { if (OB_FAIL(column_array_.at(i).serialize(buf, data_len, pos))) { STORAGE_LOG(WARN, "Fail to serialize column schema", K(ret), K(i), K_(column_array)); } - } else { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "Unexpected storage schema version", K(ret), K_(storage_schema_version)); } } return ret; @@ -830,7 +835,7 @@ int ObStorageSchema::deserialize( } else if (OB_FAIL(serialization::decode(buf, data_len, pos, storage_schema_version_))) { STORAGE_LOG(WARN, "failed to deserialize version", K(ret), K(data_len), K(pos)); } else if (STORAGE_SCHEMA_VERSION <= storage_schema_version_ - && STORAGE_SCHEMA_VERSION_V3 >= storage_schema_version_) { + && STORAGE_SCHEMA_VERSION_LATEST >= storage_schema_version_) { ObString tmp_encryption; ObString tmp_encrypt_key; LST_DO_CODE(OB_UNIS_DECODE, @@ -883,6 +888,9 @@ int ObStorageSchema::deserialize( } else if (OB_FAIL(deserialize_skip_idx_attr_array(buf, data_len, pos))) { STORAGE_LOG(WARN, "failed to deserialize skip idx attr array", K(ret)); } // TODO(@lixia.yq) need to add compat log for column_group after transfer refresh + if (OB_SUCC(ret) && storage_schema_version_ >= STORAGE_SCHEMA_VERSION_V4) { + OB_UNIS_DECODE(mv_mode_); + } if (OB_SUCC(ret)) { is_inited_ = true; @@ -969,13 +977,10 @@ int ObStorageSchema::deserialize_column_array( if (OB_FAIL(column.legacy_deserialize(buf, data_len, pos))) { STORAGE_LOG(WARN, "Fail to deserialize column schema for legacy version", K(ret)); } - } else if (STORAGE_SCHEMA_VERSION_V3 == storage_schema_version_) { + } else if (STORAGE_SCHEMA_VERSION_V3 <= storage_schema_version_) { if (OB_FAIL(column.deserialize(buf, data_len, pos))) { STORAGE_LOG(WARN, "Fail to deserialize column schema", K(ret)); } - } else { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "Unexpected storage schema version", K(ret), K_(storage_schema_version)); } if (OB_SUCC(ret) && column.orig_default_value_.get_deep_copy_size() > 0) { @@ -1422,6 +1427,10 @@ int64_t ObStorageSchema::get_serialize_size() const len += get_array_serialize_length(column_group_array_); len += get_array_serialize_length(skip_idx_attr_array_); } + if (storage_schema_version_ >= STORAGE_SCHEMA_VERSION_V4) { + OB_UNIS_ADD_LEN(mv_mode_); + } + return len; } @@ -1877,22 +1886,30 @@ int ObStorageSchema::get_multi_version_column_descs(common::ObIArray return ret; } -void ObStorageSchema::copy_from(const share::schema::ObMergeSchema &input_schema) +int ObStorageSchema::copy_from(const share::schema::ObMergeSchema &input_schema) { - is_use_bloomfilter_ = input_schema.is_use_bloomfilter(); - table_type_ = input_schema.get_table_type(); - table_mode_ = input_schema.get_table_mode_struct(); - index_type_ = input_schema.get_index_type(); - row_store_type_ = input_schema.get_row_store_type(); - schema_version_ = input_schema.get_schema_version(); - column_cnt_ = input_schema.get_column_count(); - tablet_size_ = input_schema.get_tablet_size(); - pctfree_ = input_schema.get_pctfree(); - block_size_ = input_schema.get_block_size(); - progressive_merge_round_ = input_schema.get_progressive_merge_round(); - progressive_merge_num_ = input_schema.get_progressive_merge_num(); - master_key_id_ = input_schema.get_master_key_id(); - compressor_type_ = input_schema.get_compressor_type(); + int ret = OB_SUCCESS; + + if (OB_FAIL(input_schema.get_mv_mode_struct(mv_mode_))) { + STORAGE_LOG(WARN, "Fail to get mv mode struct", K(ret)); + } else { + is_use_bloomfilter_ = input_schema.is_use_bloomfilter(); + table_type_ = input_schema.get_table_type(); + table_mode_ = input_schema.get_table_mode_struct(); + index_type_ = input_schema.get_index_type(); + row_store_type_ = input_schema.get_row_store_type(); + schema_version_ = input_schema.get_schema_version(); + column_cnt_ = input_schema.get_column_count(); + tablet_size_ = input_schema.get_tablet_size(); + pctfree_ = input_schema.get_pctfree(); + block_size_ = input_schema.get_block_size(); + progressive_merge_round_ = input_schema.get_progressive_merge_round(); + progressive_merge_num_ = input_schema.get_progressive_merge_num(); + master_key_id_ = input_schema.get_master_key_id(); + compressor_type_ = input_schema.get_compressor_type(); + } + + return ret; } int ObStorageSchema::deep_copy_str(const ObString &src, ObString &dest) diff --git a/src/storage/ob_storage_schema.h b/src/storage/ob_storage_schema.h index 6d99d8a6d6..a93f5b20d2 100644 --- a/src/storage/ob_storage_schema.h +++ b/src/storage/ob_storage_schema.h @@ -236,6 +236,18 @@ public: return share::schema::is_index_table(table_type_); } inline bool is_materialized_view() const { return share::schema::ObTableSchema::is_materialized_view(table_type_); } + inline bool is_mv_container_table() const + { + return share::schema::IS_MV_CONTAINER_TABLE == (enum share::schema::ObMVContainerTableFlag)table_mode_.mv_container_table_flag_; + } + inline bool is_mv_major_refresh() const + { + return share::schema::IS_MV_MAJOR_REFRESH == (enum share::schema::ObMVMajorRefreshFlag)mv_mode_.mv_major_refresh_flag_; + } + inline bool is_mv_major_refresh_table() const + { + return is_mv_container_table() && is_mv_major_refresh(); + } inline bool is_mlog_table() const { return share::schema::ObTableSchema::is_mlog_table(table_type_); } inline bool is_fts_index() const { return share::schema::is_fts_index(index_type_); } inline bool is_vec_index() const { return share::schema::is_vec_index(index_type_); } @@ -257,6 +269,11 @@ public: virtual inline share::schema::ObTableModeFlag get_table_mode_flag() const override { return (share::schema::ObTableModeFlag)table_mode_.mode_flag_; } virtual inline share::schema::ObTableMode get_table_mode_struct() const override { return table_mode_; } + virtual inline int get_mv_mode_struct(share::schema::ObMvMode &mv_mode) const override + { + mv_mode = mv_mode_; + return OB_SUCCESS; + } virtual inline share::schema::ObTableType get_table_type() const override { return table_type_; } virtual inline share::schema::ObIndexType get_index_type() const override { return index_type_; } const common::ObIArray &get_store_column_schemas() const { return column_array_; } @@ -305,7 +322,7 @@ public: public: static void trim(const ObCollationType type, blocksstable::ObStorageDatum &storage_datum); private: - void copy_from(const share::schema::ObMergeSchema &input_schema); + int copy_from(const share::schema::ObMergeSchema &input_schema); int deep_copy_str(const ObString &src, ObString &dest); int add_column_group(const ObStorageColumnGroupSchema &column_group); inline bool is_view_table() const { return share::schema::ObTableType::USER_VIEW == table_type_ || share::schema::ObTableType::SYSTEM_VIEW == table_type_ || share::schema::ObTableType::MATERIALIZED_VIEW == table_type_; } @@ -356,7 +373,8 @@ public: static const int64_t STORAGE_SCHEMA_VERSION = 1; static const int64_t STORAGE_SCHEMA_VERSION_V2 = 2; // add for store_column_cnt_ static const int64_t STORAGE_SCHEMA_VERSION_V3 = 3; // add for cg_group - static const int64_t STORAGE_SCHEMA_VERSION_LATEST = STORAGE_SCHEMA_VERSION_V3; + static const int64_t STORAGE_SCHEMA_VERSION_V4 = 4; + static const int64_t STORAGE_SCHEMA_VERSION_LATEST = STORAGE_SCHEMA_VERSION_V4; common::ObIAllocator *allocator_; int64_t storage_schema_version_; @@ -393,6 +411,7 @@ public: common::ObFixedArray skip_idx_attr_array_; int64_t store_column_cnt_; // NOT include virtual generated column bool has_all_column_group_; // for column store, no need to serialize + share::schema::ObMvMode mv_mode_; bool is_inited_; private: DISALLOW_COPY_AND_ASSIGN(ObStorageSchema); diff --git a/src/storage/ob_storage_util.cpp b/src/storage/ob_storage_util.cpp index 24c5e2c3a0..7073119530 100644 --- a/src/storage/ob_storage_util.cpp +++ b/src/storage/ob_storage_util.cpp @@ -566,6 +566,220 @@ int check_skip_by_monotonicity( return ret; } +int reverse_trans_version_val(common::ObDatum *datums, const int64_t count) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(nullptr == datums || count < 0)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument", K(ret), KP(datums), K(count)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < count; ++i) { + common::ObDatum &datum = datums[i]; + if (OB_UNLIKELY(datum.is_nop() || datum.is_null() || datum.get_int() > 0)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Unexpected datum value", K(ret), K(datum)); + } else { + datum.set_int(-datum.get_int()); + } + } + } + return ret; +} + +int reverse_trans_version_val(ObIVector *vector, const int64_t count) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(nullptr == vector || count < 0)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument", K(ret), KP(vector), K(count)); + } else if (OB_UNLIKELY(vector->get_format() != VectorFormat::VEC_FIXED)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Unexpected vector format for trans version col", K(ret), K(vector->get_format())); + } else { + ObFixedLengthBase *fixed_length_base = static_cast(vector); + if (OB_UNLIKELY(fixed_length_base->has_null() || fixed_length_base->get_length() != sizeof(int64_t))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Unexpected vector", K(ret), K(fixed_length_base->has_null()), K(fixed_length_base->get_length())); + } else { + int64_t *ver_ptr = reinterpret_cast(fixed_length_base->get_data()); + for (int64_t i = 0; i < count; ++i) { + ver_ptr[i] = -ver_ptr[i]; + } + } + } + return ret; +} + +const char *ObMviewScanInfo::OLD_ROW = "O"; +const char *ObMviewScanInfo::NEW_ROW = "N"; +const char *ObMviewScanInfo::FINAL_ROW = "F"; +int ObMviewScanInfo::init( + const bool is_mv_refresh_query, + const StorageScanType scan_type, + const int64_t begin_version, + const int64_t end_version, + const common::ObIArray &non_mview_filters) +{ + int ret = OB_SUCCESS; + if (non_mview_filters.count() > 0 && OB_FAIL(op_filters_.reserve(non_mview_filters.count()))) { + STORAGE_LOG(WARN, "Failed to reserve op filters", K(ret)); + } else { + is_mv_refresh_query_ = is_mv_refresh_query; + scan_type_ = scan_type; + begin_version_ = begin_version; + end_version_ = end_version; + for (int64_t i = 0; OB_SUCC(ret) && i < non_mview_filters.count(); ++i) { + if (OB_FAIL(op_filters_.push_back(non_mview_filters.at(i)))) { + STORAGE_LOG(WARN, "Failed to push back", K(ret)); + } + } + } + return ret; +} +int ObMviewScanInfo::check_and_update_version_range(const int64_t multi_version_start, common::ObVersionRange &origin_range) +{ + int ret = OB_SUCCESS; + bool is_valid = 0 == origin_range.base_version_ && + (!is_end_valid() || end_version_ == origin_range.snapshot_version_) && + (!is_begin_valid() || (begin_version_ < origin_range.snapshot_version_ && begin_version_ >= multi_version_start)); + if (OB_UNLIKELY(!is_valid)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid mview query version", K(ret), K(multi_version_start), K(origin_range), K(*this)); + } else { + if (is_begin_valid()) { + origin_range.base_version_ = begin_version_; + } + } + return ret; +} + +int decimal_or_number_to_int64(const ObDatum &datum, + const ObDatumMeta &datum_meta, + int64_t &res) +{ + int ret = OB_SUCCESS; + ObObjType ob_type = datum_meta.get_type(); + if (ObNumberType == ob_type) { + const number::ObNumber nmb(datum.get_number()); + if (OB_FAIL(nmb.extract_valid_int64_with_trunc(res))) { + STORAGE_LOG(WARN, "failed to cast number to int64", K(ret)); + } + } else if (ObDecimalIntType == ob_type) { + int32_t int_bytes = wide::ObDecimalIntConstValue::get_int_bytes_by_precision(datum_meta.precision_); + bool is_valid; + if (OB_FAIL(wide::check_range_valid_int64(datum.get_decimal_int(), int_bytes, is_valid, res))) { + STORAGE_LOG(WARN, "failed to check decimal int", K(int_bytes), K(ret)); + } else if (!is_valid) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "decimal int is not valid int64", K(ret)); + } + } else { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected type", K(ob_type), K(ret)); + } + return ret; +} + +// extract mview info from filter: [ora_rowscn > V0 and ora_rowscn <= V1] and $OLD_NEW='O|N' +int build_mview_scan_info_if_need( + const common::ObQueryFlag query_flag, + const sql::ObExprPtrIArray *op_filters, + sql::ObEvalCtx &eval_ctx, + common::ObIAllocator *alloc, + ObMviewScanInfo *&mview_scan_info) +{ + int ret = OB_SUCCESS; + StorageScanType scan_type = StorageScanType::NORMAL; + int64_t begin_version = -1; + int64_t end_version = -1; + common::ObSEArray non_mview_filters; + if (OB_UNLIKELY(nullptr == alloc || nullptr != mview_scan_info || + nullptr == op_filters || op_filters->count() < 1)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument", K(ret), KP(alloc), KP(mview_scan_info), KP(op_filters)); + } else { + sql::ObExpr *e = nullptr; + ObDatum *datum = NULL; + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(eval_ctx); + batch_info_guard.set_batch_size(1); + for (int64_t i = 0; OB_SUCC(ret) && i < op_filters->count(); ++i) { + if (OB_ISNULL(e = op_filters->at(i))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "expr is null", K(ret), K(i)); + } else if (T_OP_EQ == e->type_ && 2 == e->arg_cnt_ && + T_PSEUDO_OLD_NEW_COL == e->args_[0]->type_ && e->args_[1]->is_static_const_) { + if (OB_FAIL(e->args_[1]->eval(eval_ctx, datum))) { + STORAGE_LOG(WARN, "Failed to eval const expr", K(ret)); + } else if (0 == ObString::make_string(ObMviewScanInfo::OLD_ROW).case_compare(datum->get_string())) { + scan_type = StorageScanType::MVIEW_FIRST_DELETE; + } else if (0 == ObString::make_string(ObMviewScanInfo::NEW_ROW).case_compare(datum->get_string())) { + scan_type = StorageScanType::MVIEW_LAST_INSERT; + } else if (0 == ObString::make_string(ObMviewScanInfo::FINAL_ROW).case_compare(datum->get_string())) { + scan_type = StorageScanType::MVIEW_FINAL_ROW; + } else { + ret = OB_NOT_SUPPORTED; + STORAGE_LOG(WARN, "Not supported OLD_NEW type", K(ret), KPC(datum)); + } + } else if (OB_FAIL(non_mview_filters.push_back(e))) { + STORAGE_LOG(WARN, "Failed to push back non mview filter", K(ret), K(i)); + } else if ((T_OP_GT == e->type_ || T_OP_LE == e->type_) && 2 == e->arg_cnt_) { + sql::ObExpr *left = e->args_[0]; + sql::ObExpr *right = e->args_[1]; + int64_t rowscn = -1; + if (T_ORA_ROWSCN != left->type_ && lib::is_oracle_mode()) { + if (T_FUN_SYS_CAST == left->type_ && + 2 == left->arg_cnt_ && + T_ORA_ROWSCN == left->args_[0]->type_ ) { + left = left->args_[0]; + } else { + left = nullptr; + } + } + if (nullptr != left && + T_ORA_ROWSCN == left->type_ && (right->is_static_const_ || T_FUN_SYS_LAST_REFRESH_SCN == right->type_)) { + if (OB_FAIL(right->eval(eval_ctx, datum))) { + STORAGE_LOG(WARN, "Failed to eval const expr", K(ret)); + } else if (lib::is_oracle_mode()) { + if (OB_FAIL(decimal_or_number_to_int64(*datum, right->datum_meta_, rowscn))) { + STORAGE_LOG(WARN, "Failed to get rowscn", K(ret)); + } + } else { + rowscn = datum->get_int(); + } + if (OB_FAIL(ret)) { + } else if (T_OP_GT == e->type_) { + begin_version = rowscn; + } else { + end_version = rowscn; + } + } + } + } + } + if (OB_FAIL(ret)) { + } else if (StorageScanType::NORMAL == scan_type) { + STORAGE_LOG(INFO, "no need use mview scan", K(scan_type), K(begin_version), K(end_version)); + } else if (OB_ISNULL(mview_scan_info = OB_NEWx(ObMviewScanInfo, alloc, alloc))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "Failed to alloc memory for mview scan info", K(ret)); + } else if (OB_FAIL(mview_scan_info->init(query_flag.is_mr_mview_refresh_base_scan(), scan_type, begin_version, end_version, non_mview_filters))) { + STORAGE_LOG(WARN, "Failed to init mview scan info", K(ret)); + } + STORAGE_LOG(INFO, "[MVIEW QUERY]: build mview scan info", K(ret), KPC(mview_scan_info)); + return ret; +} + +void release_mview_scan_info(common::ObIAllocator *alloc, ObMviewScanInfo *&mview_scan_info) +{ + if (nullptr != mview_scan_info) { + mview_scan_info->~ObMviewScanInfo(); + if (nullptr != alloc) { + alloc->free(mview_scan_info); + } + mview_scan_info = nullptr; + } +} + } } diff --git a/src/storage/ob_storage_util.h b/src/storage/ob_storage_util.h index eb7ffd4615..8593deef09 100644 --- a/src/storage/ob_storage_util.h +++ b/src/storage/ob_storage_util.h @@ -16,6 +16,7 @@ #include "lib/allocator/ob_allocator.h" #include "share/datum/ob_datum_funcs.h" #include "sql/engine/expr/ob_expr.h" +#include "common/ob_common_types.h" namespace oceanbase { @@ -442,6 +443,90 @@ inline ObFilterInCmpType get_filter_in_cmp_type( return cmp_type; } +inline int reverse_trans_version_val(common::ObDatum &datum) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(datum.is_nop() || datum.is_null() || datum.get_int() > 0)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "unexpected datum value", K(ret), K(datum)); + } else { + datum.set_int(-datum.get_int()); + } + return ret; +} +int reverse_trans_version_val(common::ObDatum *datums, const int64_t count); +int reverse_trans_version_val(ObIVector *vector, const int64_t count); + +enum class StorageScanType : int8_t +{ + NORMAL = 0, + MVIEW_FIRST_DELETE = 1, + MVIEW_LAST_INSERT = 2, + MVIEW_FINAL_ROW = 3, +}; +OB_INLINE bool is_mview_scan_old_row(const StorageScanType type) +{ + return StorageScanType::MVIEW_FIRST_DELETE == type; +} +OB_INLINE bool is_mview_scan_new_row(const StorageScanType type) +{ + return StorageScanType::MVIEW_LAST_INSERT == type; +} +OB_INLINE bool is_mview_scan_final_row(const StorageScanType type) +{ + return StorageScanType::MVIEW_FINAL_ROW == type; +} +OB_INLINE bool is_mview_table_scan(const StorageScanType type) +{ + return StorageScanType::NORMAL < type && type <= StorageScanType::MVIEW_FINAL_ROW; +} +OB_INLINE bool is_mview_need_deleted_row(const StorageScanType type) +{ + return is_mview_scan_old_row(type) || is_mview_scan_final_row(type); +} + +struct ObMviewScanInfo +{ + static const char *OLD_ROW; + static const char *NEW_ROW; + static const char *FINAL_ROW; + ObMviewScanInfo(ObIAllocator *alloc) : is_mv_refresh_query_(false), + scan_type_(StorageScanType::NORMAL), + begin_version_(-1), + end_version_(-1), + op_filters_(alloc) + { + } + int init( + const bool is_mv_refresh_query, + const StorageScanType scan_type, + const int64_t begin_version, + const int64_t end_version, + const common::ObIArray &non_mview_filters); + OB_INLINE bool is_begin_valid() const { return -1 != begin_version_; } + OB_INLINE bool is_end_valid() const { return -1 != end_version_; } + OB_INLINE bool is_valid() const + { + return !(is_begin_valid() && is_end_valid() && begin_version_ >= end_version_) && + is_mview_table_scan(scan_type_); + } + int check_and_update_version_range(const int64_t multi_version_start, common::ObVersionRange &origin_range); + TO_STRING_KV(K_(is_mv_refresh_query), K_(scan_type), K_(begin_version), K_(end_version), K_(op_filters)); + bool is_mv_refresh_query_; + StorageScanType scan_type_; + // (begin_version, end_version] + int64_t begin_version_; + int64_t end_version_; + sql::ExprFixedArray op_filters_; +}; +int build_mview_scan_info_if_need( + const common::ObQueryFlag query_flag, + const sql::ObExprPtrIArray *op_filters, + sql::ObEvalCtx &eval_ctx, + common::ObIAllocator *alloc, + ObMviewScanInfo *&mview_scan_info); +void release_mview_scan_info(common::ObIAllocator *alloc, ObMviewScanInfo *&mview_scan_info); + } } diff --git a/src/storage/restore/ob_ls_restore_handler.cpp b/src/storage/restore/ob_ls_restore_handler.cpp index 45c2f3dfc2..db52750bed 100644 --- a/src/storage/restore/ob_ls_restore_handler.cpp +++ b/src/storage/restore/ob_ls_restore_handler.cpp @@ -32,6 +32,7 @@ #include "observer/ob_server_event_history_table_operator.h" #include "share/restore/ob_restore_persist_helper.h" #include "storage/tablet/ob_tablet.h" +#include "storage/high_availability/ob_rebuild_service.h" #include "storage/high_availability/ob_storage_ha_utils.h" using namespace oceanbase; @@ -2707,10 +2708,26 @@ ObLSRestoreMajorState::~ObLSRestoreMajorState() { } +ERRSIM_POINT_DEF(EN_REBUILD_BEFORE_RESTORE_MAJOR) int ObLSRestoreMajorState::do_restore() { - DEBUG_SYNC(BEFORE_RESTORE_MAJOR); int ret = OB_SUCCESS; + +#ifdef ERRSIM + if (OB_SUCCESS != EN_REBUILD_BEFORE_RESTORE_MAJOR && !ls_->is_sys_ls() && is_follower(role_)) { + // trigger follower rebuild + ObRebuildService *rebuild_service = MTL(ObRebuildService *); + const ObLSRebuildType rebuild_type(ObLSRebuildType::TRANSFER); + if (OB_FAIL(rebuild_service->add_rebuild_ls(ls_->get_ls_id(), rebuild_type))) { + LOG_WARN("[ERRSIM] failed to add rebuild ls", K(ret), K(ls_->get_ls_id()), K(rebuild_type)); + } else { + LOG_INFO("fake EN_REBUILD_BEFORE_RESTORE_MAJOR", K(ls_->get_ls_id()), K(rebuild_type)); + } + } +#endif + + DEBUG_SYNC(BEFORE_RESTORE_MAJOR); + if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); @@ -3460,4 +3477,4 @@ int ObLSRestoreStat::set_total_bytes(const int64_t bytes) } return ret; -} \ No newline at end of file +} diff --git a/src/storage/tablelock/ob_lock_table.cpp b/src/storage/tablelock/ob_lock_table.cpp index 0c4b6fcd0f..929fc2015f 100644 --- a/src/storage/tablelock/ob_lock_table.cpp +++ b/src/storage/tablelock/ob_lock_table.cpp @@ -346,7 +346,7 @@ int ObLockTable::create_tablet(const lib::Worker::CompatMode compat_mode, const } else if (OB_FAIL(get_table_schema_(tenant_id, table_schema))) { LOG_WARN("get lock table schema failed", K(ret)); } else if (OB_FAIL(create_tablet_schema.init(arena_allocator, table_schema, compat_mode, - false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3))) { + false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_LATEST))) { LOG_WARN("failed to init storage schema", KR(ret), K(table_schema)); } else if (OB_FAIL(parent_->create_ls_inner_tablet(ls_id, LS_LOCK_TABLET, diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index 3077a062b0..c7cbfd4d9e 100644 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -263,7 +263,7 @@ ObTablet::ObTablet(const bool is_external_tablet) is_external_tablet_(is_external_tablet) { #if defined(__x86_64__) && !defined(ENABLE_OBJ_LEAK_CHECK) - check_size(); + check_size(); #endif MEMSET(memtables_, 0x0, sizeof(memtables_)); } @@ -5823,6 +5823,9 @@ int ObTablet::get_kept_snapshot_info( if (OB_SUCC(ret)) { bool use_multi_version_start_on_tablet = false; old_min_reserved_snapshot = snapshot_info.snapshot_; + const bool is_new_mv_in_restore = ObSnapShotType::SNAPSHOT_FOR_MAJOR_REFRESH_MV == snapshot_info.snapshot_type_ && + 0 == snapshot_info.snapshot_; + // if exist new mv in restore, use special snapshot info if (min_reserved_snapshot_on_ls > 0) { snapshot_info.update_by_smaller_snapshot(ObStorageSnapshotInfo::SNAPSHOT_FOR_LS_RESERVED, min_reserved_snapshot_on_ls); snapshot_info.update_by_smaller_snapshot(ObStorageSnapshotInfo::SNAPSHOT_FOR_MIN_MEDIUM, min_medium_snapshot); @@ -5833,13 +5836,12 @@ int ObTablet::get_kept_snapshot_info( // if not sync ls_reserved_snapshot yet, should use multi_version_start on tablet use_multi_version_start_on_tablet = true; } - if (use_multi_version_start_on_tablet) { + if (use_multi_version_start_on_tablet || is_new_mv_in_restore) { snapshot_info.snapshot_type_ = ObStorageSnapshotInfo::SNAPSHOT_MULTI_VERSION_START_ON_TABLET; snapshot_info.snapshot_ = get_multi_version_start(); } // snapshot info should smaller than snapshot on tablet snapshot_info.update_by_smaller_snapshot(ObStorageSnapshotInfo::SNAPSHOT_ON_TABLET, get_snapshot_version()); - const int64_t current_time = common::ObTimeUtility::fast_current_time(); if (current_time - (snapshot_info.snapshot_ / 1000 /*use microsecond here*/) > 2_hour) { if (REACH_TENANT_TIME_INTERVAL(10_s)) { diff --git a/src/storage/tablet/ob_tablet_create_mds_helper.cpp b/src/storage/tablet/ob_tablet_create_mds_helper.cpp index 58e58ac8fd..028946d6eb 100644 --- a/src/storage/tablet/ob_tablet_create_mds_helper.cpp +++ b/src/storage/tablet/ob_tablet_create_mds_helper.cpp @@ -602,7 +602,7 @@ int ObTabletCreateMdsHelper::convert_schemas( LOG_WARN("failed to allocate storage schema", KR(ret), K(table_schema)); } else if (FALSE_IT(create_tablet_schema = new (create_tablet_schema_ptr)ObCreateTabletSchema())) { } else if (OB_FAIL(create_tablet_schema->init(arg.allocator_, table_schema, compat_mode, - false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3))) { + false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_LATEST))) { LOG_WARN("failed to init storage schema", KR(ret), K(table_schema)); } else if (OB_FAIL(arg.create_tablet_schemas_.push_back(create_tablet_schema))) { LOG_WARN("failed to push back table schema", KR(ret), K(create_tablet_schema)); diff --git a/src/storage/tablet/ob_tablet_table_store.cpp b/src/storage/tablet/ob_tablet_table_store.cpp index 29fc643b99..b7e50573ca 100644 --- a/src/storage/tablet/ob_tablet_table_store.cpp +++ b/src/storage/tablet/ob_tablet_table_store.cpp @@ -1567,6 +1567,7 @@ int ObTabletTableStore::inner_replace_remote_major_sstable_( ObITable *old_table = nullptr; ObSSTableMetaHandle old_sst_meta_hdl; ObSSTableMetaHandle new_sst_meta_hdl; + bool has_backup_macro = false; if (OB_ISNULL(new_table)) { ret = OB_INVALID_ARGUMENT; @@ -1574,16 +1575,15 @@ int ObTabletTableStore::inner_replace_remote_major_sstable_( } else if (old_store.major_tables_.empty()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("no major table exist", K(ret), K(old_store), KPC(new_table)); - } else if (OB_FAIL(static_cast(new_table)->get_meta(new_sst_meta_hdl))) { - LOG_WARN("failed to get new sstable meta handle", K(ret), KPC(new_table)); - } else if (new_sst_meta_hdl.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup()) { + } else if (OB_FAIL(ObTableStoreUtil::check_has_backup_macro_block(new_table, has_backup_macro))) { + LOG_WARN("failed to check new table has backup macro block", K(ret), KPC(new_table)); + } else if (has_backup_macro) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("new table still has backup macro block", K(ret), KPC(new_table), K(new_sst_meta_hdl)); + LOG_WARN("new table still has backup macro block", K(ret), KPC(new_table)); } else if (OB_FAIL(old_store.major_tables_.get_all_tables(old_tables_array))) { LOG_WARN("failed to get major tables from old store", K(ret), K(old_store)); } - bool has_backup_macro = false; for (int64_t idx = 0; OB_SUCC(ret) && idx < old_tables_array.count(); ++idx) { old_table = old_tables_array.at(idx); if (OB_ISNULL(old_table)) { @@ -1591,14 +1591,10 @@ int ObTabletTableStore::inner_replace_remote_major_sstable_( LOG_WARN("get unexpected null table", K(ret), K(old_store)); } else if (OB_FAIL(ObTableStoreUtil::check_has_backup_macro_block(old_table, has_backup_macro))) { LOG_WARN("failed to check table has backup macro block", K(ret), KPC(old_table)); - } else if (!has_backup_macro) { + } else if (old_table->get_key() != new_table->get_key() || !has_backup_macro) { if (OB_FAIL(new_tables_array.push_back(old_table))) { LOG_WARN("failed to push back", K(ret)); } - } else if (old_table->get_key() != new_table->get_key()) { - // Major compaction is not allowed during restore, only one major should be exist. - ret = OB_ERR_UNEXPECTED; - LOG_WARN("too many major sstable exist", K(ret), KPC(old_table), KPC(new_table), K(old_store)); } else if (OB_FAIL(new_tables_array.push_back(new_table))) { LOG_WARN("failed to push back", K(ret)); } else { @@ -2356,13 +2352,6 @@ int ObTabletTableStore::check_new_major_sstable_can_be_accepted_( if (OB_FAIL(check_new_sstable_can_be_accepted_(ddl_sstables, major_table))) { LOG_WARN("failed to check accept the compacted sstable", K(ret), K(ddl_sstables), KPC(major_table)); } - } else if (OB_FAIL(old_store.major_tables_.get_all_tables(major_tables))) { - LOG_WARN("failed to get major tables from old store", K(ret), K(old_store)); - } else if (major_tables.empty()) { - // remote major sstable first be created in table store. - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("major sstable exist", K(ret), K(old_store), KPC(major_table)); } return ret; diff --git a/src/storage/tx/ob_dup_table_tablets.cpp b/src/storage/tx/ob_dup_table_tablets.cpp index c1799a57a0..f3c8e467b8 100644 --- a/src/storage/tx/ob_dup_table_tablets.cpp +++ b/src/storage/tx/ob_dup_table_tablets.cpp @@ -3218,10 +3218,17 @@ int ObTenantDupTabletSchemaHelper::get_all_dup_tablet_set_(TabletIDSet &tablet_s DUP_TABLE_LOG(WARN, "get table schemas in tenant failed", K(ret)); } else { for (int64_t i = 0; OB_SUCCESS == ret && i < table_schemas.count(); i++) { + bool is_restore = false; bool is_duplicated = false; const ObSimpleTableSchemaV2 *table_schema = table_schemas.at(i); - if (OB_FAIL(table_schema->check_is_duplicated(schema_guard, is_duplicated))) { - DUP_TABLE_LOG(WARN, "check duplicate failed", K(ret)); + if (OB_FAIL(schema_guard.check_tenant_is_restore(table_schema->get_tenant_id(), is_restore))) { + DUP_TABLE_LOG(WARN, "fail to check tenant is restore", K(ret)); + } else if (is_restore) { + is_duplicated = false; + } else if (table_schema->is_duplicate_table()) { + is_duplicated = true; + } + if (OB_FAIL(ret)) { } else if (is_duplicated) { ObArray tablet_id_arr; if (OB_FAIL(table_schema->get_tablet_ids(tablet_id_arr))) { diff --git a/src/storage/tx/ob_multi_data_source_printer.cpp b/src/storage/tx/ob_multi_data_source_printer.cpp index 984c688a3c..ec308e4e78 100644 --- a/src/storage/tx/ob_multi_data_source_printer.cpp +++ b/src/storage/tx/ob_multi_data_source_printer.cpp @@ -51,6 +51,8 @@ const char *ObMultiDataSourcePrinter::to_str_mds_type(const ObTxDataSourceType & TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, CHANGE_TABLET_TO_TABLE_MDS); TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, TABLET_SPLIT); TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, TABLET_BINDING); + TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, MV_PUBLISH_SCN); + TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, MV_NOTICE_SAFE); TRX_ENUM_CASE_TO_STR(ObTxDataSourceType, MAX_TYPE); } diff --git a/src/storage/tx/ob_tx_log.cpp b/src/storage/tx/ob_tx_log.cpp index a3a8e72258..fc10ab2d7f 100644 --- a/src/storage/tx/ob_tx_log.cpp +++ b/src/storage/tx/ob_tx_log.cpp @@ -42,8 +42,8 @@ ObTxLogTypeChecker::need_replay_barrier(const ObTxLogType log_type, || data_source_type == ObTxDataSourceType::START_TRANSFER_OUT_PREPARE || data_source_type == ObTxDataSourceType::FINISH_TRANSFER_OUT || data_source_type == ObTxDataSourceType::TABLET_SPLIT - || data_source_type == ObTxDataSourceType::TABLET_BINDING) { - + || data_source_type == ObTxDataSourceType::TABLET_BINDING + || data_source_type == ObTxDataSourceType::MV_NOTICE_SAFE) { barrier_flag = logservice::ObReplayBarrierType::PRE_BARRIER; } else if (data_source_type == ObTxDataSourceType::FINISH_TRANSFER_IN diff --git a/src/storage/tx/wrs/ob_black_list.cpp b/src/storage/tx/wrs/ob_black_list.cpp index d4173239c7..791ee6c570 100644 --- a/src/storage/tx/wrs/ob_black_list.cpp +++ b/src/storage/tx/wrs/ob_black_list.cpp @@ -267,7 +267,7 @@ int ObBLService::do_black_list_check_(sqlclient::ObMySQLResult *result) } else if (ls_info.is_leader() && check_need_skip_leader_(bl_key.get_tenant_id())) { // cannot add leader into blacklist need_remove = true; - } else if (ls_info.weak_read_scn_ == 0) { + } else if (ls_info.weak_read_scn_ == 0 && ls_info.migrate_status_ == OB_MIGRATION_STATUS_NONE) { // log stream is initializing, should't be put into blacklist need_remove = true; } else if (OB_FAIL(OB_TS_MGR.get_gts(bl_key.get_tenant_id(), NULL, gts_scn))) { diff --git a/src/storage/tx/wrs/ob_black_list.h b/src/storage/tx/wrs/ob_black_list.h index 883c58af84..5be9991b5a 100644 --- a/src/storage/tx/wrs/ob_black_list.h +++ b/src/storage/tx/wrs/ob_black_list.h @@ -95,13 +95,15 @@ public: } int compare(const ObBLKey &other) const { - int ret = 0; - uint64_t hash_1 = hash(); - uint64_t hash_2 = other.hash(); - if (hash_1 > hash_2) { - ret = 1; - } else if (hash_1 < hash_2) { - ret = -1; + int ret = server_.compare(other.server_); + if (0 == ret) { + if (tenant_id_ > other.tenant_id_) { + ret = 1; + } else if (tenant_id_ < other.tenant_id_) { + ret = -1; + } else { + ret = ls_id_.compare(other.ls_id_); + } } return ret; } diff --git a/src/storage/tx_storage/ob_ls_service.cpp b/src/storage/tx_storage/ob_ls_service.cpp index 9b3a5d1499..5b31ef115a 100644 --- a/src/storage/tx_storage/ob_ls_service.cpp +++ b/src/storage/tx_storage/ob_ls_service.cpp @@ -420,6 +420,7 @@ int ObLSService::inner_create_ls_(const share::ObLSID &lsid, const ObMigrationStatus &migration_status, const ObLSRestoreStatus &restore_status, const SCN &create_scn, + const ObMajorMVMergeInfo &major_mv_merge_info, const ObLSStoreFormat &store_format, ObLS *&ls) { @@ -438,6 +439,7 @@ int ObLSService::inner_create_ls_(const share::ObLSID &lsid, migration_status, restore_status, create_scn, + major_mv_merge_info, store_format, rs_reporter_))) { LOG_WARN("fail to init ls", K(ret), K(lsid)); @@ -517,6 +519,7 @@ int ObLSService::create_ls(const obrpc::ObCreateLSArg &arg) common_arg.restore_status_ = get_restore_status_by_tenant_role_(arg.get_tenant_info().get_tenant_role()); common_arg.create_type_ = get_create_type_by_tenant_role_(arg.get_tenant_info().get_tenant_role()); common_arg.need_create_inner_tablet_ = need_create_inner_tablets_(arg); + common_arg.major_mv_merge_info_ = arg.get_major_mv_merge_info(); if (OB_FAIL(create_ls_(common_arg, mig_arg))) { LOG_WARN("create ls failed", K(ret), K(arg)); @@ -918,6 +921,7 @@ int ObLSService::replay_create_ls_(const int64_t ls_epoch, const ObLSMeta &ls_me migration_status, restore_status, ls_meta.get_clog_checkpoint_scn(), + ls_meta.get_major_mv_merge_info(), ls_meta.get_store_format(), ls))) { LOG_WARN("fail to inner create ls", K(ret), K(ls_meta.ls_id_)); @@ -1285,6 +1289,7 @@ int ObLSService::create_ls_(const ObCreateLSCommonArg &arg, arg.migration_status_, arg.restore_status_, arg.create_scn_, + arg.major_mv_merge_info_, ls_store_format, ls))) { LOG_WARN("create ls failed", K(ret), K(arg.ls_id_), K(ls_store_format)); @@ -1367,6 +1372,7 @@ int ObLSService::create_ls_for_ha( common_arg.restore_status_ = restore_status; common_arg.task_id_ = task_id; common_arg.need_create_inner_tablet_ = false; + common_arg.major_mv_merge_info_.reset(); if (OB_FAIL(create_ls_(common_arg, arg))) { LOG_WARN("failed to create ls", K(ret), K(arg)); diff --git a/src/storage/tx_storage/ob_ls_service.h b/src/storage/tx_storage/ob_ls_service.h index e8d0266825..ed9f22585b 100644 --- a/src/storage/tx_storage/ob_ls_service.h +++ b/src/storage/tx_storage/ob_ls_service.h @@ -19,6 +19,7 @@ #include "storage/ob_storage_rpc.h" #include "storage/ls/ob_ls_meta_package.h" // ObLSMetaPackage #include "share/resource_limit_calculator/ob_resource_limit_calculator.h" +#include "storage/mview/ob_major_mv_merge_info.h" namespace oceanbase { @@ -200,6 +201,7 @@ private: ObLSRestoreStatus restore_status_; share::ObTaskId task_id_; bool need_create_inner_tablet_; + storage::ObMajorMVMergeInfo major_mv_merge_info_; }; int create_ls_(const ObCreateLSCommonArg &arg, @@ -211,6 +213,7 @@ private: const ObMigrationStatus &migration_status, const share::ObLSRestoreStatus &restore_status, const share::SCN &create_scn, + const ObMajorMVMergeInfo &major_mv_merge_info, const ObLSStoreFormat &store_format, ObLS *&ls); int inner_del_ls_(ObLS *&ls); diff --git a/src/storage/tx_table/ob_tx_table.cpp b/src/storage/tx_table/ob_tx_table.cpp index c36f7a6223..1f73345807 100644 --- a/src/storage/tx_table/ob_tx_table.cpp +++ b/src/storage/tx_table/ob_tx_table.cpp @@ -302,7 +302,7 @@ int ObTxTable::create_ctx_tablet_( if (OB_FAIL(get_ctx_table_schema_(tenant_id, table_schema))) { LOG_WARN("get ctx table schema failed", K(ret)); } else if (OB_FAIL(create_tablet_schema.init(arena_allocator, table_schema, compat_mode, - false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3))) { + false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_LATEST))) { LOG_WARN("failed to init storage schema", KR(ret), K(table_schema)); } else if (OB_FAIL(ls_->create_ls_inner_tablet(ls_id, LS_TX_CTX_TABLET, @@ -429,7 +429,7 @@ int ObTxTable::create_data_tablet_(const uint64_t tenant_id, if (OB_FAIL(get_data_table_schema_(tenant_id, table_schema))) { LOG_WARN("get data table schema failed", K(ret)); } else if (OB_FAIL(create_tablet_schema.init(arena_allocator, table_schema, compat_mode, - false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3))) { + false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_LATEST))) { LOG_WARN("failed to init storage schema", KR(ret), K(table_schema)); } else if (OB_FAIL(ls_->create_ls_inner_tablet(ls_id, LS_TX_DATA_TABLET, diff --git a/tools/deploy/mysql_test/test_suite/duplicate_table/r/mysql/test_4x_duplicate_table.result b/tools/deploy/mysql_test/test_suite/duplicate_table/r/mysql/test_4x_duplicate_table.result index 4a0de07a7f..b1b5af0d72 100644 --- a/tools/deploy/mysql_test/test_suite/duplicate_table/r/mysql/test_4x_duplicate_table.result +++ b/tools/deploy/mysql_test/test_suite/duplicate_table/r/mysql/test_4x_duplicate_table.result @@ -32,7 +32,7 @@ dup_t1 CREATE TABLE `dup_t1` ( `c2` int(11) NOT NULL, `c3` int(11) DEFAULT NULL, PRIMARY KEY (`c1`, `c2`) -) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' +) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' DUPLICATE_READ_CONSISTENCY = 'STRONG' show create table dup_t2; Table Create Table dup_t2 CREATE TABLE `dup_t2` ( @@ -40,7 +40,7 @@ dup_t2 CREATE TABLE `dup_t2` ( `d2` int(11) NOT NULL, `d3` int(11) DEFAULT NULL, PRIMARY KEY (`d1`, `d2`) -) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' +) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' DUPLICATE_READ_CONSISTENCY = 'STRONG' insert into t1 values(1, 1, 1); insert into t1 values(2, 2, 2); insert into t1 values(3, 3, 3); diff --git a/tools/deploy/mysql_test/test_suite/duplicate_table/r/mysql/test_4x_duplicate_table_basic_lease.result b/tools/deploy/mysql_test/test_suite/duplicate_table/r/mysql/test_4x_duplicate_table_basic_lease.result index d435a6d1b9..edb33d2c96 100644 --- a/tools/deploy/mysql_test/test_suite/duplicate_table/r/mysql/test_4x_duplicate_table_basic_lease.result +++ b/tools/deploy/mysql_test/test_suite/duplicate_table/r/mysql/test_4x_duplicate_table_basic_lease.result @@ -32,7 +32,7 @@ dup_t1 CREATE TABLE `dup_t1` ( `c2` int(11) NOT NULL, `c3` int(11) DEFAULT NULL, PRIMARY KEY (`c1`, `c2`) -) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' +) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' DUPLICATE_READ_CONSISTENCY = 'STRONG' show create table dup_t2; Table Create Table dup_t2 CREATE TABLE `dup_t2` ( @@ -40,7 +40,7 @@ dup_t2 CREATE TABLE `dup_t2` ( `d2` int(11) NOT NULL, `d3` int(11) DEFAULT NULL, PRIMARY KEY (`d1`, `d2`) -) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' +) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' DUPLICATE_READ_CONSISTENCY = 'STRONG' duplicate_scope 1 duplicate_scope diff --git a/tools/deploy/mysql_test/test_suite/duplicate_table/r/mysql/test_4x_duplicate_table_basic_tablet_set.result b/tools/deploy/mysql_test/test_suite/duplicate_table/r/mysql/test_4x_duplicate_table_basic_tablet_set.result index 2376c92c01..c4d35234b7 100644 --- a/tools/deploy/mysql_test/test_suite/duplicate_table/r/mysql/test_4x_duplicate_table_basic_tablet_set.result +++ b/tools/deploy/mysql_test/test_suite/duplicate_table/r/mysql/test_4x_duplicate_table_basic_tablet_set.result @@ -29,7 +29,7 @@ dup_t1 CREATE TABLE `dup_t1` ( `c2` int(11) NOT NULL, `c3` int(11) DEFAULT NULL, PRIMARY KEY (`c1`, `c2`) -) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' +) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' DUPLICATE_READ_CONSISTENCY = 'STRONG' show create table dup_t2; Table Create Table dup_t2 CREATE TABLE `dup_t2` ( @@ -37,7 +37,7 @@ dup_t2 CREATE TABLE `dup_t2` ( `d2` int(11) NOT NULL, `d3` int(11) DEFAULT NULL, PRIMARY KEY (`d1`, `d2`) -) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' +) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' DUPLICATE_READ_CONSISTENCY = 'STRONG' show create table dup_t3; Table Create Table dup_t3 CREATE TABLE `dup_t3` ( @@ -45,7 +45,7 @@ dup_t3 CREATE TABLE `dup_t3` ( `d2` int(11) DEFAULT NULL, `d3` int(11) DEFAULT NULL, PRIMARY KEY (`d1`) -) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' +) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' DUPLICATE_READ_CONSISTENCY = 'STRONG' partition by hash(d1) (partition `p0`, partition `p1`, @@ -59,7 +59,7 @@ dup_t4 CREATE TABLE `dup_t4` ( `d2` int(11) DEFAULT NULL, `d3` int(11) DEFAULT NULL, PRIMARY KEY (`d1`) -) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' +) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' DUPLICATE_READ_CONSISTENCY = 'STRONG' partition by hash(d1) (partition `p0`, partition `p1`, diff --git a/tools/deploy/mysql_test/test_suite/duplicate_table/r/mysql/test_4x_duplicate_table_basic_tablets.result b/tools/deploy/mysql_test/test_suite/duplicate_table/r/mysql/test_4x_duplicate_table_basic_tablets.result index dd4612871b..9153b69589 100644 --- a/tools/deploy/mysql_test/test_suite/duplicate_table/r/mysql/test_4x_duplicate_table_basic_tablets.result +++ b/tools/deploy/mysql_test/test_suite/duplicate_table/r/mysql/test_4x_duplicate_table_basic_tablets.result @@ -34,7 +34,7 @@ dup_t1 CREATE TABLE `dup_t1` ( `c2` int(11) NOT NULL, `c3` int(11) DEFAULT NULL, PRIMARY KEY (`c1`, `c2`) -) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' +) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' DUPLICATE_READ_CONSISTENCY = 'STRONG' show create table dup_t2; Table Create Table dup_t2 CREATE TABLE `dup_t2` ( @@ -42,7 +42,7 @@ dup_t2 CREATE TABLE `dup_t2` ( `d2` int(11) NOT NULL, `d3` int(11) DEFAULT NULL, PRIMARY KEY (`d1`, `d2`) -) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' +) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' DUPLICATE_READ_CONSISTENCY = 'STRONG' show create table dup_t3; Table Create Table dup_t3 CREATE TABLE `dup_t3` ( @@ -50,7 +50,7 @@ dup_t3 CREATE TABLE `dup_t3` ( `d2` int(11) DEFAULT NULL, `d3` int(11) DEFAULT NULL, PRIMARY KEY (`d1`) -) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' +) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' DUPLICATE_READ_CONSISTENCY = 'STRONG' partition by hash(d1) (partition `p0`, partition `p1`, @@ -64,7 +64,7 @@ dup_t4 CREATE TABLE `dup_t4` ( `d2` int(11) DEFAULT NULL, `d3` int(11) DEFAULT NULL, PRIMARY KEY (`d1`) -) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' +) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = NUM BLOCK_SIZE = SIZE USE_BLOOM_FILTER = FALSE TABLET_SIZE = SIZE PCTFREE = 10 DUPLICATE_SCOPE = 'CLUSTER' DUPLICATE_READ_CONSISTENCY = 'STRONG' partition by hash(d1) (partition `p0`, partition `p1`, diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result index bf23014f0a..9b6172835b 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result @@ -6217,7 +6217,7 @@ MASTER_ROLLBACK_SEG varchar(128) NO MASTER_LINK varchar(128) NO REWRITE_ENABLED varchar(1) NO REWRITE_CAPABILITY varchar(9) NO -REFRESH_MODE varchar(6) NO +REFRESH_MODE varchar(32) NO REFRESH_METHOD varchar(8) NO BUILD_MODE varchar(9) NO FAST_REFRESHABLE varchar(18) NO diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result index 0ebf956eaa..d82e2c9ffc 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result @@ -8952,7 +8952,7 @@ MASTER_ROLLBACK_SEG varchar(128) NO MASTER_LINK varchar(128) NO REWRITE_ENABLED varchar(1) NO REWRITE_CAPABILITY varchar(9) NO -REFRESH_MODE varchar(6) NO +REFRESH_MODE varchar(32) NO REFRESH_METHOD varchar(8) NO BUILD_MODE varchar(9) NO FAST_REFRESHABLE varchar(18) NO @@ -8995,7 +8995,7 @@ MASTER_ROLLBACK_SEG varchar(128) NO MASTER_LINK varchar(128) NO REWRITE_ENABLED varchar(1) NO REWRITE_CAPABILITY varchar(9) NO -REFRESH_MODE varchar(6) NO +REFRESH_MODE varchar(32) NO REFRESH_METHOD varchar(8) NO BUILD_MODE varchar(9) NO FAST_REFRESHABLE varchar(18) NO diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result index f980498377..5f3e342f73 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result @@ -888,6 +888,7 @@ local_session_vars longtext YES NULL duplicate_read_consistency bigint(20) NO 0 index_params varchar(256) NO micro_index_clustered tinyint(4) NO 0 +mv_mode bigint(20) NO 0 select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_core_all_table; IF(count(*) >= 0, 1, 0) 1 @@ -1824,6 +1825,7 @@ local_session_vars longtext YES NULL duplicate_read_consistency bigint(20) NO 0 index_params varchar(256) NO micro_index_clustered tinyint(4) NO 0 +mv_mode bigint(20) NO 0 select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_table; IF(count(*) >= 0, 1, 0) 1 @@ -2905,6 +2907,9 @@ tablet_change_checkpoint_scn bigint(20) unsigned NO NULL transfer_scn bigint(20) unsigned NO NULL tx_blocked bigint(20) NO NULL required_data_disk_size bigint(20) NO 0 +mv_major_merge_scn bigint(20) unsigned NO 0 +mv_publish_scn bigint(20) unsigned NO 0 +mv_safe_scn bigint(20) unsigned NO 0 select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_ls_info; IF(count(*) >= 0, 1, 0) 1 diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result index c83645fbe0..e0fb2fe48c 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result @@ -961,6 +961,7 @@ local_session_vars longtext YES NULL duplicate_read_consistency bigint(20) NO 0 index_params varchar(256) NO micro_index_clustered tinyint(4) NO 0 +mv_mode bigint(20) NO 0 select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_core_all_table; IF(count(*) >= 0, 1, 0) 1 @@ -2719,6 +2720,7 @@ local_session_vars longtext YES NULL duplicate_read_consistency bigint(20) NO 0 index_params varchar(256) NO micro_index_clustered tinyint(4) NO 0 +mv_mode bigint(20) NO 0 select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_table; IF(count(*) >= 0, 1, 0) 1 @@ -2814,6 +2816,7 @@ local_session_vars longtext YES NULL duplicate_read_consistency bigint(20) YES 0 index_params varchar(256) YES micro_index_clustered tinyint(4) YES 0 +mv_mode bigint(20) YES 0 select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_table_history; IF(count(*) >= 0, 1, 0) 1 @@ -6498,6 +6501,9 @@ tablet_change_checkpoint_scn bigint(20) unsigned NO NULL transfer_scn bigint(20) unsigned NO NULL tx_blocked bigint(20) NO NULL required_data_disk_size bigint(20) NO 0 +mv_major_merge_scn bigint(20) unsigned NO 0 +mv_publish_scn bigint(20) unsigned NO 0 +mv_safe_scn bigint(20) unsigned NO 0 select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_ls_info; IF(count(*) >= 0, 1, 0) 1 diff --git a/unittest/storage/init_basic_struct.h b/unittest/storage/init_basic_struct.h index ed0d3eb35b..bdbc4be877 100644 --- a/unittest/storage/init_basic_struct.h +++ b/unittest/storage/init_basic_struct.h @@ -66,10 +66,11 @@ int __attribute__((weak)) gen_create_ls_arg(const int64_t tenant_id, arg.reset(); lib::Worker::CompatMode compat_mode = lib::Worker::CompatMode::MYSQL; palf::PalfBaseInfo palf_base_info; + ObMajorMVMergeInfo major_merge_info; if (OB_FAIL(tenant_info.init(tenant_id, share::PRIMARY_TENANT_ROLE))) { STORAGE_LOG(WARN, "failed to init tenant info", KR(ret), K(tenant_id)); - } else if (OB_FAIL(arg.init(tenant_id, ls_id, replica_type, property, tenant_info, create_scn, compat_mode, false, palf_base_info))) { - STORAGE_LOG(WARN, "failed to init arg", KR(ret), K(tenant_id), K(ls_id), K(tenant_info), K(create_scn), K(compat_mode), K(palf_base_info)); + } else if (OB_FAIL(arg.init(tenant_id, ls_id, replica_type, property, tenant_info, create_scn, compat_mode, false, palf_base_info, major_merge_info))) { + STORAGE_LOG(WARN, "failed to init arg", KR(ret), K(tenant_id), K(ls_id), K(tenant_info), K(create_scn), K(compat_mode), K(palf_base_info), K(major_merge_info)); } return ret; } diff --git a/unittest/storage/mock_ob_log_handler.h b/unittest/storage/mock_ob_log_handler.h index a31795d93b..1f874353f0 100644 --- a/unittest/storage/mock_ob_log_handler.h +++ b/unittest/storage/mock_ob_log_handler.h @@ -196,6 +196,14 @@ public: UNUSEDx(member_list, paxos_replica_num, learner_list); return OB_SUCCESS; } + int get_stable_membership(palf::LogConfigVersion &config_version, + common::ObMemberList &member_list, + int64_t &paxos_replica_num, + common::GlobalLearnerList &learner_list) const + { + UNUSEDx(config_version, member_list, paxos_replica_num, learner_list); + return OB_SUCCESS; + } int get_max_lsn(palf::LSN &lsn) const { UNUSED(lsn);