diff --git a/deps/oblib/src/common/object/ob_obj_compare.h b/deps/oblib/src/common/object/ob_obj_compare.h index d93287bf67..f7c080eed0 100644 --- a/deps/oblib/src/common/object/ob_obj_compare.h +++ b/deps/oblib/src/common/object/ob_obj_compare.h @@ -259,7 +259,15 @@ public: } const double l = obj1.get_double(); const double r = obj2.get_double(); - if (l == r || fabs(l - r) < p) { + if (isnan(l) || isnan(r)) { + if (isnan(l) && isnan(r)) { + ret = 0; + } else if (isnan(l)) { + ret = 1; + } else { + ret = -1; + } + } else if (l == r || fabs(l - r) < p) { ret = 0; } else { ret = (l < r ? -1 : 1); diff --git a/deps/oblib/src/lib/lock/ob_spin_rwlock.h b/deps/oblib/src/lib/lock/ob_spin_rwlock.h index 34962a22fe..22847cb53b 100644 --- a/deps/oblib/src/lib/lock/ob_spin_rwlock.h +++ b/deps/oblib/src/lib/lock/ob_spin_rwlock.h @@ -84,6 +84,37 @@ private: DISALLOW_COPY_AND_ASSIGN(SpinRLockGuard); }; +class SpinRLockManualGuard +{ +public: + explicit SpinRLockManualGuard() + : lock_(nullptr), ret_(OB_SUCCESS) + { + } + ~SpinRLockManualGuard() + { + if (OB_LIKELY(OB_SUCCESS == ret_) && OB_NOT_NULL(lock_)) { + if (OB_UNLIKELY(OB_SUCCESS != (ret_ = lock_->unlock()))) { + COMMON_LOG_RET(WARN, ret_, "Fail to unlock, ", K_(ret)); + } else { + lock_ = nullptr; + } + } + } + void lock(SpinRWLock &lock) { + lock_ = &lock; + if (OB_UNLIKELY(OB_SUCCESS != (ret_ = lock_->rdlock()))) { + COMMON_LOG_RET(WARN, ret_, "Fail to read lock, ", K_(ret)); + } + } + inline int get_ret() const { return ret_; } +private: + SpinRWLock *lock_; + int ret_; +private: + DISALLOW_COPY_AND_ASSIGN(SpinRLockManualGuard); +}; + class SpinWLockGuard { public: diff --git a/deps/oblib/src/lib/ob_errno.h b/deps/oblib/src/lib/ob_errno.h index 10e86476a8..8abe3193fe 100644 --- a/deps/oblib/src/lib/ob_errno.h +++ b/deps/oblib/src/lib/ob_errno.h @@ -376,6 +376,7 @@ constexpr int OB_ERR_TOO_MANY_PREFIX_DECLARE = -7424; constexpr int OB_ERR_XPATH_INVALID_NODE = -7426; constexpr int OB_ERR_XPATH_NO_NODE = -7427; constexpr int OB_ERR_DUP_DEF_NAMESPACE = -7431; +constexpr int OB_ERR_INVALID_VECTOR_DIM = -7600; constexpr int OB_PACKET_CLUSTER_ID_NOT_MATCH = -8004; constexpr int OB_TENANT_ID_NOT_MATCH = -8005; constexpr int OB_URI_ERROR = -9001; diff --git a/deps/oblib/src/lib/stat/ob_latch_define.h b/deps/oblib/src/lib/stat/ob_latch_define.h index e8d1f083ed..00a5556de7 100644 --- a/deps/oblib/src/lib/stat/ob_latch_define.h +++ b/deps/oblib/src/lib/stat/ob_latch_define.h @@ -317,7 +317,6 @@ LATCH_DEF(DAS_ASYNC_RPC_LOCK, 290, "das wait remote response lock", LATCH_FIFO, LATCH_DEF(CLOG_CKPT_RWLOCK, 291, "clog checkpoint rwlock", LATCH_READ_PREFER, 2000, 0, true) LATCH_DEF(REWRITE_RULE_ITEM_LOCK, 292, "rewrite rule item lock", LATCH_FIFO, 2000, 0, true) -LATCH_DEF(TENANT_MGR_TENANT_BUCKET_LOCK, 290, "tenant mgr tenant bucket lock", LATCH_READ_PREFER, INT64_MAX, 0, false) LATCH_DEF(SRS_LOCK, 293, "srs lock", LATCH_READ_PREFER, 2000, 0, false) LATCH_DEF(DDL_EXECUTE_LOCK, 294, "ddl execute lock", LATCH_FIFO, 2000, 0, true) LATCH_DEF(TENANT_IO_CONFIG_LOCK, 295, "tenant io config lock", LATCH_FIFO, 2000, 0, true) @@ -369,8 +368,9 @@ LATCH_DEF(SQL_AUDIT, 335, "sql audit release second level queue lock", LATCH_FIF LATCH_DEF(S2_PHY_BLOCK_LOCK, 336, "s2 phy block lock", LATCH_FIFO, INT64_MAX, 0, false) LATCH_DEF(S2_MEM_BLOCK_LOCK, 337, "s2 mem block lock", LATCH_FIFO, INT64_MAX, 0, false) +LATCH_DEF(TENANT_MGR_TENANT_BUCKET_LOCK, 338, "tenant mgr tenant bucket lock", LATCH_READ_PREFER, INT64_MAX, 0, false) -LATCH_DEF(LATCH_END, 338, "latch end", LATCH_FIFO, 2000, 0, true) +LATCH_DEF(LATCH_END, 339, "latch end", LATCH_FIFO, 2000, 0, true) #endif diff --git a/deps/oblib/src/lib/statistic_event/ob_stat_event.h b/deps/oblib/src/lib/statistic_event/ob_stat_event.h index 91de67b931..2017fa89a3 100644 --- a/deps/oblib/src/lib/statistic_event/ob_stat_event.h +++ b/deps/oblib/src/lib/statistic_event/ob_stat_event.h @@ -21,6 +21,7 @@ * @param summary_in_session Stat recorded for user session process mark this flag true. * @param can_visible Indicate whether this stat can be queried on gv$sysstat and gv$sesstat. * @param enable Indicate whether this stat is enabled. Marked it false it you merely need it as an placeholder. + * NOTICE: do not reuse stat id or rename stat event! */ // NETWORK diff --git a/deps/oblib/src/lib/wait_event/ob_wait_event.h b/deps/oblib/src/lib/wait_event/ob_wait_event.h index 36b8ae8f89..73054d0150 100644 --- a/deps/oblib/src/lib/wait_event/ob_wait_event.h +++ b/deps/oblib/src/lib/wait_event/ob_wait_event.h @@ -22,8 +22,9 @@ * @param class Every wait event belongs to a class of wait event on deps/oblib/src/lib/wait_event/ob_wait_class.h * @param is_phy Indicate whether this wait event can be nested. true for most cases. * @param enable Means whether this wait event is enabled. Marked it false it you merely need it as an placeholder. + * NOTICE: do not reuse wait event id or rename wait event! */ -// USER_IO & SYSTEM_IO +// USER_IO & SYSTEM_IO 10001-11999 WAIT_EVENT_DEF(NULL_EVENT, 10000, "", "", "", "", OTHER, true, true) WAIT_EVENT_DEF(DB_FILE_DATA_READ, 10001, "db file data read", "fd", "offset", "size", USER_IO, true, true) WAIT_EVENT_DEF(DB_FILE_DATA_INDEX_READ, 10002, "db file data `index read", "fd", "offset", "size", USER_IO, true, true) @@ -44,18 +45,18 @@ WAIT_EVENT_DEF(PALF_WRITE, 11017, "palf write", "fd", "offset", "size", SYSTEM_I WAIT_EVENT_DEF(OBJECT_STORAGE_WRITE, 11018, "object storage write", "fd", "offset", "size", SYSTEM_IO, true, false) WAIT_EVENT_DEF(OBJECT_STORAGE_READ, 11019, "object storage read", "fd", "offset", "size", SYSTEM_IO, true, false) -// SCHEDULER +// SCHEDULER 12001-12999 WAIT_EVENT_DEF(OMT_WAIT, 12001, "sched wait", "req type", "req start timestamp", "wait start timestamp", SCHEDULER, true, false) WAIT_EVENT_DEF(OMT_IDLE, 12002, "sched idle", "wait start timestamp", "", "", IDLE, true, true) -// NETWORK +// NETWORK 13000-13999 WAIT_EVENT_DEF(SYNC_RPC, 13000, "sync rpc", "pcode", "size", "", NETWORK, true, true) WAIT_EVENT_DEF(MYSQL_RESPONSE_WAIT_CLIENT, 13001, "mysql response wait client", "", "", "", NETWORK, true, true) WAIT_EVENT_DEF(DAS_ASYNC_RPC_LOCK_WAIT, 13002, "das wait remote response", "", "", "", NETWORK, true, true) WAIT_EVENT_DEF(ASYNC_EXTERNAL_TABLE_LOCK_WAIT, 13003, "external table wait remote response", "", "", "", NETWORK, true, true) WAIT_EVENT_DEF(NETWORK_QUEUE_WAIT, 13004, "wait for network request in queue", "pcode", "retry_times", "", NETWORK, true, true) -// APPLICATION +// APPLICATION 14001-14999 WAIT_EVENT_DEF(MT_READ_LOCK_WAIT,14001,"memstore read lock wait","lock","waiter","owner",APPLICATION,false, true) WAIT_EVENT_DEF(MT_WRITE_LOCK_WAIT,14002,"memstore write lock wait","lock","waiter","owner",APPLICATION,false, false) WAIT_EVENT_DEF(ROW_LOCK_WAIT,14003,"row lock wait","lock","waiter","owner",APPLICATION,false, false) @@ -117,7 +118,6 @@ WAIT_EVENT_DEF(ASYNC_COMMITTING_WAIT, 16018, "async commiting wait", "", "", "", WAIT_EVENT_DEF(OBCDC_PART_MGR_SCHEMA_VERSION_WAIT, 18000, "oblog part mgr schema version wait", "", "", "", CONCURRENCY, true, true) WAIT_EVENT_DEF(SYNC_GET_GTS_WAIT, 18101, "sync get gts timestamp wait", "address", "", "", CONCURRENCY, true, true) -// sleep WAIT_EVENT_DEF(BANDWIDTH_THROTTLE_SLEEP, 20000, "sleep: bandwidth throttle sleep wait", "sleep_interval", "", "", CONCURRENCY, true, true) WAIT_EVENT_DEF(DTL_PROCESS_CHANNEL_SLEEP, 20001, "sleep: dtl process channel sleep wait", "sleep_interval", "", "", CONCURRENCY, true, true) WAIT_EVENT_DEF(DTL_DESTROY_CHANNEL_SLEEP, 20002, "sleep: dtl destroy channel sleep wait", "sleep_interval", "", "", CONCURRENCY, true, true) @@ -125,15 +125,11 @@ WAIT_EVENT_DEF(STORAGE_WRITING_THROTTLE_SLEEP, 20003, "sleep: storage writing th WAIT_EVENT_DEF(STORAGE_AUTOINC_FETCH_RETRY_SLEEP, 20004, "sleep: tablet autoinc fetch new range retry wait", "sleep_interval", "", "", CONCURRENCY, true, true) WAIT_EVENT_DEF(STORAGE_AUTOINC_FETCH_CONFLICT_SLEEP, 20005, "sleep: tablet autoinc fetch new range conflict wait", "sleep_interval", "", "", CONCURRENCY, true, true) WAIT_EVENT_DEF(STORAGE_HA_FINISH_TRANSFER, 20006, "sleep: finish transfer sleep wait", "sleep_interval", "", "", CONCURRENCY, true, true) -WAIT_EVENT_DEF(GARBAGE_COLLECTOR_SLEEP, 20007, "sleep: wait log callback sleep wait", "sleep_interval", "", "", CONCURRENCY, true, true) - - -// logservice WAIT_EVENT_DEF(LOG_EXTERNAL_STORAGE_IO_TASK_WAIT, 20007, "latch: log external storage io task wait", "", "", "", CONCURRENCY, true, true) WAIT_EVENT_DEF(LOG_EXTERNAL_STORAGE_HANDLER_RW_WAIT, 20008, "latch: log external storage handler rw wait", "", "", "", CONCURRENCY, true, false) WAIT_EVENT_DEF(LOG_EXTERNAL_STORAGE_HANDLER_WAIT, 20009, "latch: log external storage handler spin wait", "", "", "", CONCURRENCY, true, false) -// share storage +// share storage 21001-21999 WAIT_EVENT_DEF(ZONE_STORAGE_MANAGER_LOCK_WAIT, 21001, "latch: zone storage manager maintaince lock wait", "address", "number", "tries", CONCURRENCY, true, false) WAIT_EVENT_DEF(ZONE_STORAGE_INFO_RW_LOCK_WAIT, 21002, "latch: zone storage infos rw lock wait", "address", "number", "tries", CONCURRENCY, true, false) WAIT_EVENT_DEF(DEVICE_MANIFEST_RW_LOCK_WAIT, 21003, "latch: device_manifest rw lock wait", "address", "number", "tries", CONCURRENCY, true, false) @@ -149,13 +145,17 @@ WAIT_EVENT_DEF(TIERED_BLOCK_WRITE_LOCAL, 21012, "tiered block write local", "add WAIT_EVENT_DEF(TIERED_BLOCK_READ_REMOTE, 21013, "tiered block read remote", "address", "", "", CONCURRENCY, true, false) WAIT_EVENT_DEF(TIERED_BLOCK_READ_LOCAL, 21014, "tiered block read local", "address", "", "", CONCURRENCY, true, false) -// inner sql +// inner sql 30000-30099 WAIT_EVENT_DEF(INNER_SQL_EXEC_WAIT, 30000, "exec inner sql wait", "wait inner sql class", "inner session id", "", OTHER, true, true) WAIT_EVENT_DEF(INNER_SESSION_IDLE_WAIT, 30001, "inner session wait to be called", "inner session id", "parent session id", "", IDLE, true, false) -// CONFIGURATION +// CONFIGURATION 30100-30999 WAIT_EVENT_DEF(WAIT_REFRESH_SCHEMA, 30100, "sleep: wait refresh schema", "sleep_interval", "schema_version", "", CONFIGURATION, true, true) WAIT_EVENT_DEF(PALF_THROTTLING, 30101, "palf throttling sleep", "sleep_interval", "", "", USER_IO, false, true) +WAIT_EVENT_DEF(SLOG_NORMAL_RETRY_SLEEP, 30102, "sleep: slog has io error and retrying", "sleep_interval", "", "", CONCURRENCY, true, true) + +// sleep 31000-31999 +WAIT_EVENT_DEF(GARBAGE_COLLECTOR_SLEEP, 31000, "sleep: wait log callback sleep wait", "sleep_interval", "", "", CONCURRENCY, true, true) // END. DO NOT MODIFY. WAIT_EVENT_DEF(WAIT_EVENT_DEF_END, 99999, "event end", "", "", "", OTHER, false, true) diff --git a/deps/oblib/src/rpc/obmysql/ob_mysql_protocol_processor.cpp b/deps/oblib/src/rpc/obmysql/ob_mysql_protocol_processor.cpp index 7dd43d57c6..625063f00a 100644 --- a/deps/oblib/src/rpc/obmysql/ob_mysql_protocol_processor.cpp +++ b/deps/oblib/src/rpc/obmysql/ob_mysql_protocol_processor.cpp @@ -54,7 +54,7 @@ int ObMysqlProtocolProcessor::do_decode(ObSMConnection& conn, ObICSMemPool& pool // go backward with MySQL packet header length start -= header_size; next_read_bytes = delta_len; - } else if (conn.is_in_authed_phase()) { + } else if (conn.is_in_authed_phase() || conn.is_in_auth_switch_phase()) { if (OB_FAIL(decode_body(pool, start, pktlen, pktseq, pkt))) { LOG_ERROR("fail to decode_body", K(sessid), K(pktseq), K(ret)); } diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h index 14b5673332..ead6e5c414 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h @@ -108,6 +108,7 @@ PCODE_DEF(OB_CHECK_SERVER_FOR_ADDING_SERVER, 0x153) PCODE_DEF(OB_GET_SERVER_RESOURCE_INFO, 0x154) PCODE_DEF(OB_NOTIFY_SWITCH_LEADER, 0x155) PCODE_DEF(OB_GET_TENANT_TABLET_COUNT, 0x156) +PCODE_DEF(OB_CHECK_SERVER_MACHINE_STATUS, 0x157) // remote interrupt call PCODE_DEF(OB_REMOTE_INTERRUPT_CALL, 0x1EE) @@ -1154,6 +1155,7 @@ PCODE_DEF(OB_CAL_UNIT_PHY_RESOURCE, 0x1622) PCODE_DEF(OB_CAL_STANDBY_TENANT_PHY_RESOURCE, 0x1623) //PCODE_DEF(OB_UPDATE_MVIEW_REFERENCE_TABLE_STATUS, 0x1624) +//PCODE_DEF(OB_DO_EVENT_DDL, 0x1625) //**** 注意：在此行之前增加新的RPC ID ****** // diff --git a/mittest/CMakeLists.txt b/mittest/CMakeLists.txt index eb18cc6589..2e98a7460a 100644 --- a/mittest/CMakeLists.txt +++ b/mittest/CMakeLists.txt @@ -1,4 +1,6 @@ -add_library(mit_env env/ob_simple_server_helper.cpp) +set(MIT_SRCS env/ob_simple_server_helper.cpp) + +add_library(mit_env ${MIT_SRCS}) target_include_directories(mit_env PUBLIC ${CMAKE_SOURCE_DIR}/unittest ${CMAKE_SOURCE_DIR}/mittest) diff --git a/mittest/env/ob_simple_server_helper.cpp b/mittest/env/ob_simple_server_helper.cpp index a100aca740..843142c245 100644 --- a/mittest/env/ob_simple_server_helper.cpp +++ b/mittest/env/ob_simple_server_helper.cpp @@ -311,6 +311,57 @@ int SimpleServerHelper::freeze(uint64_t tenant_id, ObLSID ls_id, ObTabletID tabl return ret; } +int SimpleServerHelper::freeze_tx_data(uint64_t tenant_id, ObLSID ls_id) +{ + int ret = OB_SUCCESS; + MTL_SWITCH(tenant_id) { + ObLSHandle ls_handle; + if (OB_FAIL(MTL(ObLSService*)->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { + } else { + storage::checkpoint::ObCheckpointExecutor *checkpoint_executor = ls_handle.get_ls()->get_checkpoint_executor(); + ObTxDataMemtableMgr *tx_data_memtable_mgr + = dynamic_cast( + dynamic_cast( + checkpoint_executor->handlers_[logservice::TRANS_SERVICE_LOG_BASE_TYPE]) + ->common_checkpoints_[storage::checkpoint::ObCommonCheckpointType::TX_DATA_MEMTABLE_TYPE]); + if (OB_ISNULL(tx_data_memtable_mgr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("checkpoint obj is null", KR(ret)); + } else if (OB_FAIL(tx_data_memtable_mgr->flush(share::SCN::max_scn(), + checkpoint::INVALID_TRACE_ID))) { + } else { + usleep(10 * 1000 * 1000); + } + } + } + return ret; +} + +int SimpleServerHelper::freeze_tx_ctx(uint64_t tenant_id, ObLSID ls_id) +{ + int ret = OB_SUCCESS; + MTL_SWITCH(tenant_id) { + ObLSHandle ls_handle; + if (OB_FAIL(MTL(ObLSService*)->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { + } else { + storage::checkpoint::ObCheckpointExecutor *checkpoint_executor = ls_handle.get_ls()->get_checkpoint_executor(); + ObTxCtxMemtable *tx_ctx_memtable + = dynamic_cast( + dynamic_cast( + checkpoint_executor->handlers_[logservice::TRANS_SERVICE_LOG_BASE_TYPE]) + ->common_checkpoints_[storage::checkpoint::ObCommonCheckpointType::TX_CTX_MEMTABLE_TYPE]); + if (OB_ISNULL(tx_ctx_memtable)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("checkpoint obj is null", KR(ret)); + } else if (OB_FAIL(tx_ctx_memtable->flush(share::SCN::max_scn(), 0))) { + } else { + usleep(10 * 1000 * 1000); + } + } + } + return ret; +} + int SimpleServerHelper::wait_flush_finish(uint64_t tenant_id, ObLSID ls_id, ObTabletID tablet_id) { int ret = OB_SUCCESS; @@ -695,10 +746,9 @@ int SimpleServerHelper::ls_reboot(uint64_t tenant_id, ObLSID ls_id) LOG_INFO("ls_reboot", K(tenant_id), K(ls_id)); int ret = OB_SUCCESS; auto print_mgr_state = [](ObLS *ls) { - auto state = ls->ls_tx_svr_.mgr_->state_; + const ObTxLSStateMgr &state_mgr = ls->ls_tx_svr_.mgr_->tx_ls_state_mgr_; LOG_INFO("print ls ctx mgr state:", K(ls->get_ls_id()), - "ctx_mgr_state", state, - K(ObLSTxCtxMgr::State::state_str(state))); + K(state_mgr)); }; auto func = [tenant_id, ls_id, print_mgr_state] () { int ret = OB_SUCCESS; diff --git a/mittest/env/ob_simple_server_helper.h b/mittest/env/ob_simple_server_helper.h index 41d136355d..6bf4bcbf86 100644 --- a/mittest/env/ob_simple_server_helper.h +++ b/mittest/env/ob_simple_server_helper.h @@ -70,6 +70,8 @@ public: static int get_ls_end_scn(uint64_t tenant_id, ObLSID ls_id, SCN &end_scn); static int wait_replay_advance(uint64_t tenant_id, ObLSID ls_id, SCN end_scn); static int wait_checkpoint_newest(uint64_t tenant_id, ObLSID ls_id); + static int freeze_tx_ctx(uint64_t tenant_id, ObLSID ls_id); + static int freeze_tx_data(uint64_t tenant_id, ObLSID ls_id); static int wait_tx(uint64_t tenant_id, ObLSID ls_id, ObTransID tx_id, ObTxState tx_state); static int wait_tx_exit(uint64_t tenant_id, ObLSID ls_id, ObTransID tx_id); static int wait_flush_finish(uint64_t tenant_id, ObLSID ls_id, ObTabletID tablet_id); diff --git a/mittest/logservice/env/ob_simple_log_cluster_env.cpp b/mittest/logservice/env/ob_simple_log_cluster_env.cpp index a80b742a41..f7aa574634 100644 --- a/mittest/logservice/env/ob_simple_log_cluster_env.cpp +++ b/mittest/logservice/env/ob_simple_log_cluster_env.cpp @@ -1025,8 +1025,9 @@ int ObSimpleLogClusterTestEnv::raw_write(PalfHandleImplGuard &leader, do { usleep(10); ret = (leader.palf_handle_impl_)->submit_group_log(opts, lsn, buf, buf_len); - if (OB_SUCC(ret)) { + if (OB_SUCC(ret) || OB_ERR_OUT_OF_LOWER_BOUND == ret) { PALF_LOG(INFO, "raw_write success", KR(ret), K(lsn)); + ret = OB_SUCCESS; } else { if (REACH_TIME_INTERVAL(100 * 1000)) { PALF_LOG(WARN, "raw_write failed", KR(ret)); @@ -1167,6 +1168,10 @@ int ObSimpleLogClusterTestEnv::read_and_submit_group_log(PalfHandleImplGuard &le PALF_LOG(WARN, "iterator next failed", K(ret), K(iterator_raw_write)); } else if (OB_FAIL(iterator_raw_write.get_entry(buffer, nbytes, scn, lsn, is_raw_write))) { PALF_LOG(WARN, "iterator get_entry failed", K(ret), K(iterator_raw_write), K(is_raw_write)); + } else if (lsn >= start_lsn && is_raw_write != true) { + ret = OB_ERR_UNEXPECTED; + PALF_LOG(ERROR, "iterator get_entry failed, is_raw_write must be true", K(ret), K(iterator_raw_write), K(is_raw_write), + K(lsn), K(start_lsn)); } } } diff --git a/mittest/logservice/test_ob_simple_log_single_replica_func.cpp b/mittest/logservice/test_ob_simple_log_single_replica_func.cpp index 3d5f6f2cc4..568de143bc 100644 --- a/mittest/logservice/test_ob_simple_log_single_replica_func.cpp +++ b/mittest/logservice/test_ob_simple_log_single_replica_func.cpp @@ -1960,6 +1960,35 @@ TEST_F(TestObSimpleLogClusterSingleReplica, test_raw_read) } } +TEST_F(TestObSimpleLogClusterSingleReplica, test_raw_write_concurrent_lsn) +{ + SET_CASE_LOG_FILE(TEST_NAME, "test_raw_write_concurrent_lsn"); + int64_t id = ATOMIC_AAF(&palf_id_, 1); + OB_LOGGER.set_log_level("TRACE"); + int64_t leader_idx = 0; + PalfHandleImplGuard leader; + PalfHandleImplGuard raw_write_leader; + EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader)); + PalfHandleImpl *palf_handle_impl = leader.palf_handle_impl_; + const int64_t id_raw_write = ATOMIC_AAF(&palf_id_, 1); + EXPECT_EQ(OB_SUCCESS, create_paxos_group(id_raw_write, leader_idx, raw_write_leader)); + EXPECT_EQ(OB_SUCCESS, change_access_mode_to_raw_write(raw_write_leader)); + + EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, leader_idx, MAX_LOG_BASE_TYPE)); + SCN max_scn1 = leader.palf_handle_impl_->get_max_scn(); + LSN end_pos_of_log1 = leader.palf_handle_impl_->get_max_lsn(); + EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, leader.palf_handle_impl_->get_max_lsn())); + + std::thread submit_log_t1([&]() { + EXPECT_EQ(OB_ITER_END, read_and_submit_group_log(leader, raw_write_leader)); + }); + std::thread submit_log_t2([&]() { + EXPECT_EQ(OB_ITER_END, read_and_submit_group_log(leader, raw_write_leader)); + }); + submit_log_t1.join(); + submit_log_t2.join(); +} + } // namespace unittest } // namespace oceanbase diff --git a/mittest/mtlenv/storage/test_memtable_v2.cpp b/mittest/mtlenv/storage/test_memtable_v2.cpp index ea66ef08ab..b6b243658a 100644 --- a/mittest/mtlenv/storage/test_memtable_v2.cpp +++ b/mittest/mtlenv/storage/test_memtable_v2.cpp @@ -257,8 +257,9 @@ public: { ObUndoAction undo(from, to); ObPartTransCtx *tx_ctx = store_ctx->mvcc_acc_ctx_.tx_ctx_; - EXPECT_EQ(OB_SUCCESS, - tx_ctx->ctx_tx_data_.add_undo_action(undo)); + ObTxDataGuard tx_data_guard; + EXPECT_EQ(OB_SUCCESS, tx_ctx->ls_tx_ctx_mgr_->get_tx_table()->alloc_tx_data(tx_data_guard)); + EXPECT_EQ(OB_SUCCESS, tx_ctx->insert_undo_action_to_tx_table_(undo, tx_data_guard, SCN::min_scn())); ObMemtableCtx *mt_ctx = store_ctx->mvcc_acc_ctx_.mem_ctx_; ObTxCallbackList &cb_list = mt_ctx->trans_mgr_.callback_list_; for (ObMvccRowCallback *iter = (ObMvccRowCallback *)(cb_list.get_guard()->get_next()); @@ -3624,9 +3625,12 @@ int ObLSTxCtxMgr::init(const int64_t tenant_id, } else { if (OB_FAIL(ls_tx_ctx_map_.init(lib::ObMemAttr(tenant_id, "LSTxCtxMgr")))) { TRANS_LOG(WARN, "ls_tx_ctx_map_ init fail", KR(ret)); + } else if (OB_FAIL(tx_ls_state_mgr_.init(ls_id))) { + TRANS_LOG(WARN, "init tx_ls_state_mgr_ failed", KR(ret)); + } else if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::START))) { + TRANS_LOG(WARN, "start ls_tx_ctx_mgr failed",K(ret),K(tx_ls_state_mgr_)); } else { is_inited_ = true; - state_ = State::F_ALL_BLOCKED; tenant_id_ = tenant_id; ls_id_ = ls_id; tx_table_ = tx_table; diff --git a/mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp b/mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp index ceed7ab1b4..70c8793a59 100644 --- a/mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp +++ b/mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp @@ -1113,6 +1113,7 @@ TEST_F(TestMultiVersionMerge, test_merge_with_multi_trans) for (int64_t i = 1; i <= 4; i++) { ObTxData *tx_data = new ObTxData(); + ASSERT_EQ(OB_SUCCESS, tx_data->init_tx_op()); transaction::ObTransID tx_id = i; // fill in data @@ -1276,6 +1277,7 @@ TEST_F(TestMultiVersionMerge, test_merge_with_multi_trans_can_compact) for (int64_t i = 1; i <= 5; i++) { ObTxData *tx_data = new ObTxData(); + ASSERT_EQ(OB_SUCCESS, tx_data->init_tx_op()); transaction::ObTransID tx_id = i; // fill in data @@ -1439,6 +1441,7 @@ TEST_F(TestMultiVersionMerge, test_merge_with_multi_trans_can_not_compact) for (int64_t i = 1; i <= 5; i++) { ObTxData *tx_data = new ObTxData(); + ASSERT_EQ(OB_SUCCESS, tx_data->init_tx_op()); transaction::ObTransID tx_id = i; // fill in data @@ -3338,6 +3341,7 @@ TEST_F(TestMultiVersionMerge, test_running_trans_cross_macro_with_abort_sql_seq) ASSERT_NE(nullptr, tx_table = tx_table_guard.get_tx_table()); ObTxData *tx_data = new ObTxData(); + ASSERT_EQ(OB_SUCCESS, tx_data->init_tx_op()); transaction::ObTransID tx_id = 1; // fill in data diff --git a/mittest/mtlenv/storage/test_tenant_meta_mem_mgr.cpp b/mittest/mtlenv/storage/test_tenant_meta_mem_mgr.cpp index df1b0671e3..81407087e2 100644 --- a/mittest/mtlenv/storage/test_tenant_meta_mem_mgr.cpp +++ b/mittest/mtlenv/storage/test_tenant_meta_mem_mgr.cpp @@ -17,6 +17,7 @@ #define protected public #define private public +#include "lib/alloc/memory_dump.h" #include "storage/tablet/ob_tablet_persister.h" #include "storage/meta_mem/ob_tenant_meta_mem_mgr.h" #include "storage/meta_mem/ob_tablet_leak_checker.h" @@ -1730,6 +1731,34 @@ TEST_F(TestTenantMetaMemMgr, test_show_limit) lib::set_tenant_memory_limit(MTL_ID(), before_tenant_mem); } +TEST_F(TestTenantMetaMemMgr, test_normal_tablet_buffer_fragment) +{ + static const int64_t tablet_cnt = 155000; + ObTabletHandle *tablets = new ObTabletHandle[tablet_cnt]; + const int64_t before_tenant_mem = lib::get_tenant_memory_limit(MTL_ID()); + const int64_t this_case_tenant_mem = 3 * 1024 * 1024 * 1024L; /* 3GB */ + lib::set_tenant_memory_limit(MTL_ID(), this_case_tenant_mem); + for (int64_t i = 0; i < tablet_cnt; ++i) { + ObTabletHandle tablet_handle; + ASSERT_EQ(OB_SUCCESS, MTL(ObTenantMetaMemMgr *)->acquire_tablet(ObTabletPoolType::TP_NORMAL, tablets[i])); + } + ObMallocAllocator::get_instance()->print_tenant_memory_usage(MTL_ID()); + ObMemoryDump::get_instance().init(); + auto task = ObMemoryDump::get_instance().alloc_task(); + task->type_ = STAT_LABEL; + ObMemoryDump::get_instance().push(task); + usleep(1000000); + ObTenantCtxAllocatorGuard ta = ObMallocAllocator::get_instance()->get_tenant_ctx_allocator(MTL_ID(), ObCtxIds::META_OBJ_CTX_ID); + double fragment_rate = 1.0 * (ta->get_hold() - ta->get_used()) / ta->get_hold(); + std::cout << "hold: " << ta->get_hold() << " used: " << ta->get_used() << " limit: " << ta->get_limit() << " fragment_rate: " << fragment_rate << std::endl; + ASSERT_TRUE(fragment_rate < 0.04); + for (int64_t i = 0; i < tablet_cnt; ++i) { + tablets[i].reset(); + } + delete [] tablets; + lib::set_tenant_memory_limit(MTL_ID(), before_tenant_mem); +} + } // end namespace storage } // end namespace oceanbase diff --git a/mittest/mtlenv/test_htable_lock.cpp b/mittest/mtlenv/test_htable_lock.cpp index 4266d37f49..69be1cfa67 100644 --- a/mittest/mtlenv/test_htable_lock.cpp +++ b/mittest/mtlenv/test_htable_lock.cpp @@ -401,7 +401,7 @@ TEST_F(TestHTableLock, concurrent_shared_lock) TEST_F(TestHTableLock, concurrent_exclusive_lock) { EXPECT_EQ(OB_SYS_TENANT_ID, MTL_ID()); - const uint64_t thread_cnt = 1024; + const uint64_t thread_cnt = 100; const uint64_t fake_table_id = 1; ObString key = ObString::make_string("k1"); const int64_t start = ObTimeUtility::current_time(); @@ -416,7 +416,7 @@ TEST_F(TestHTableLock, concurrent_exclusive_lock) TEST_F(TestHTableLock, concurrent_exclusive_shared_lock) { EXPECT_EQ(OB_SYS_TENANT_ID, MTL_ID()); - const uint64_t xthread_cnt = 1024; + const uint64_t xthread_cnt = 100; const uint64_t sthread_cnt = 1024; const uint64_t fake_table_id = 1; ObString key = ObString::make_string("k1"); diff --git a/mittest/mtlenv/test_tx_data_table.cpp b/mittest/mtlenv/test_tx_data_table.cpp index aafd7770c1..ddada3ce6d 100644 --- a/mittest/mtlenv/test_tx_data_table.cpp +++ b/mittest/mtlenv/test_tx_data_table.cpp @@ -250,6 +250,7 @@ void TestTxDataTable::insert_tx_data_() tx_data_guard.reset(); ASSERT_EQ(OB_SUCCESS, tx_data_table_.alloc_tx_data(tx_data_guard, false)); ASSERT_NE(nullptr, tx_data = tx_data_guard.tx_data()); + ASSERT_EQ(OB_SUCCESS, tx_data->init_tx_op()); // fill in data tx_data->tx_id_ = tx_id; @@ -287,6 +288,7 @@ void TestTxDataTable::insert_rollback_tx_data_() ObTxData *tx_data = nullptr; ASSERT_EQ(OB_SUCCESS, tx_data_table_.alloc_tx_data(tx_data_guard, false)); ASSERT_NE(nullptr, tx_data = tx_data_guard.tx_data()); + ASSERT_EQ(OB_SUCCESS, tx_data->init_tx_op()); // fill in data tx_data->tx_id_ = tx_id; @@ -410,7 +412,9 @@ void TestTxDataTable::do_basic_test() int64_t inserted_cnt_after_pre_process = freezing_memtable->get_tx_data_count(); ASSERT_EQ(inserted_cnt_before_pre_process + 1, inserted_cnt_after_pre_process); - ASSERT_EQ(OB_SUCCESS, freezing_memtable->get_split_ranges(nullptr, nullptr, range_cnt, range_array)); + ObStoreRange input_range; + input_range.set_whole_range(); + ASSERT_EQ(OB_SUCCESS, freezing_memtable->get_split_ranges(input_range, range_cnt, range_array)); int64_t pre_range_end_key = 0; for (int i = 0; i < range_cnt; i++) { auto &range = range_array[i]; @@ -464,23 +468,24 @@ void TestTxDataTable::do_undo_status_test() ObTxDataGuard tx_data_guard; ASSERT_EQ(OB_SUCCESS, tx_data_table_.alloc_tx_data(tx_data_guard, false)); ASSERT_NE(nullptr, tx_data = tx_data_guard.tx_data()); + ASSERT_EQ(OB_SUCCESS, tx_data->init_tx_op()); tx_data->tx_id_ = rand(); for (int i = 1; i <= 1001; i++) { transaction::ObUndoAction undo_action(ObTxSEQ(10 * (i + 1), 0), ObTxSEQ(10 * i, 0)); ASSERT_EQ(OB_SUCCESS, tx_data->add_undo_action(&tx_table_, undo_action)); } - ASSERT_EQ(1000 / TX_DATA_UNDO_ACT_MAX_NUM_PER_NODE + 1, tx_data->undo_status_list_.undo_node_cnt_); + ASSERT_EQ(1000 / TX_DATA_UNDO_ACT_MAX_NUM_PER_NODE + 1, tx_data->op_guard_->get_undo_status_list().undo_node_cnt_); { transaction::ObUndoAction undo_action(ObTxSEQ(10000000, 0), ObTxSEQ(10,0)); ASSERT_EQ(OB_SUCCESS, tx_data->add_undo_action(&tx_table_, undo_action)); } - STORAGETEST_LOG(INFO, "", K(tx_data->undo_status_list_)); - ASSERT_EQ(1, tx_data->undo_status_list_.head_->size_); - ASSERT_EQ(nullptr, tx_data->undo_status_list_.head_->next_); - ASSERT_EQ(1, tx_data->undo_status_list_.undo_node_cnt_); + STORAGETEST_LOG(INFO, "", K(tx_data->op_guard_->get_undo_status_list())); + ASSERT_EQ(1, tx_data->op_guard_->get_undo_status_list().head_->size_); + ASSERT_EQ(nullptr, tx_data->op_guard_->get_undo_status_list().head_->next_); + ASSERT_EQ(1, tx_data->op_guard_->get_undo_status_list().undo_node_cnt_); } { @@ -490,23 +495,24 @@ void TestTxDataTable::do_undo_status_test() ObTxDataGuard tx_data_guard; ASSERT_EQ(OB_SUCCESS, tx_data_table_.alloc_tx_data(tx_data_guard, false)); ASSERT_NE(nullptr, tx_data = tx_data_guard.tx_data()); + ASSERT_EQ(OB_SUCCESS, tx_data->init_tx_op()); tx_data->tx_id_ = rand(); for (int i = 1; i <= 14; i++) { transaction::ObUndoAction undo_action(ObTxSEQ(i + 1,0), ObTxSEQ(i,0)); ASSERT_EQ(OB_SUCCESS, tx_data->add_undo_action(&tx_table_, undo_action)); } - ASSERT_EQ(2, tx_data->undo_status_list_.undo_node_cnt_); + ASSERT_EQ(2, tx_data->op_guard_->get_undo_status_list().undo_node_cnt_); { transaction::ObUndoAction undo_action(ObTxSEQ(15, 0), ObTxSEQ(7,0)); ASSERT_EQ(OB_SUCCESS, tx_data->add_undo_action(&tx_table_, undo_action)); } - STORAGETEST_LOG(INFO, "", K(tx_data->undo_status_list_)); - ASSERT_EQ(7, tx_data->undo_status_list_.head_->size_); - ASSERT_EQ(nullptr, tx_data->undo_status_list_.head_->next_); - ASSERT_EQ(1, tx_data->undo_status_list_.undo_node_cnt_); + STORAGETEST_LOG(INFO, "", K(tx_data->op_guard_->get_undo_status_list())); + ASSERT_EQ(7, tx_data->op_guard_->get_undo_status_list().head_->size_); + ASSERT_EQ(nullptr, tx_data->op_guard_->get_undo_status_list().head_->next_); + ASSERT_EQ(1, tx_data->op_guard_->get_undo_status_list().undo_node_cnt_); } } @@ -516,6 +522,7 @@ void TestTxDataTable::test_serialize_with_action_cnt_(int cnt) ObTxDataGuard tx_data_guard; ASSERT_EQ(OB_SUCCESS, tx_data_table_.alloc_tx_data(tx_data_guard, false)); ASSERT_NE(nullptr, tx_data = tx_data_guard.tx_data()); + ASSERT_EQ(OB_SUCCESS, tx_data->init_tx_op()); tx_data->tx_id_ = transaction::ObTransID(269381); tx_data->commit_version_.convert_for_logservice(ObTimeUtil::current_time_ns()); tx_data->end_scn_.convert_for_logservice(ObTimeUtil::current_time_ns()); @@ -532,7 +539,7 @@ void TestTxDataTable::test_serialize_with_action_cnt_(int cnt) } else { node_cnt = cnt / 7 + 1; } - ASSERT_EQ(node_cnt, tx_data->undo_status_list_.undo_node_cnt_); + ASSERT_EQ(node_cnt, tx_data->op_guard_->get_undo_status_list().undo_node_cnt_); char *buf = nullptr; ObArenaAllocator allocator; @@ -652,6 +659,7 @@ void TestTxDataTable::do_repeat_insert_test() { ObTxDataGuard tx_data_guard; ASSERT_EQ(OB_SUCCESS, tx_data_table_.alloc_tx_data(tx_data_guard, false)); ASSERT_NE(nullptr, tx_data = tx_data_guard.tx_data()); + ASSERT_EQ(OB_SUCCESS, tx_data->init_tx_op()); // fill in data tx_data->tx_id_ = tx_id; diff --git a/mittest/multi_replica/CMakeLists.txt b/mittest/multi_replica/CMakeLists.txt index cfd22591ab..c6d014c1e4 100644 --- a/mittest/multi_replica/CMakeLists.txt +++ b/mittest/multi_replica/CMakeLists.txt @@ -39,3 +39,4 @@ ob_unittest_multi_replica(test_max_commit_ts_read_from_dup_table) ob_unittest_multi_replica(test_mds_replay_from_ctx_table) ob_unittest_multi_replica_longer_timeout(test_multi_transfer_tx) ob_unittest_multi_replica(test_ob_direct_load_inc_log) +ob_unittest_multi_replica(test_tx_ls_state_switch) diff --git a/mittest/multi_replica/env/ob_multi_replica_test_base.cpp b/mittest/multi_replica/env/ob_multi_replica_test_base.cpp index 28d0834705..8ee6588853 100644 --- a/mittest/multi_replica/env/ob_multi_replica_test_base.cpp +++ b/mittest/multi_replica/env/ob_multi_replica_test_base.cpp @@ -898,7 +898,7 @@ int ObMultiReplicaTestBase::check_tenant_exist(bool &bool_ret, const char *tenan } // namespace unittest } // namespace oceanbase -int ::oceanbase::omt::ObWorkerProcessor::process_err_test() +OB_NOINLINE int ::oceanbase::omt::ObWorkerProcessor::process_err_test() { int ret = OB_SUCCESS; diff --git a/mittest/multi_replica/test_tx_ls_state_switch.cpp b/mittest/multi_replica/test_tx_ls_state_switch.cpp new file mode 100644 index 0000000000..326dd4df23 --- /dev/null +++ b/mittest/multi_replica/test_tx_ls_state_switch.cpp @@ -0,0 +1,422 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#define USING_LOG_PREFIX SERVER +#define protected public +#define private public + +#include "env/ob_fast_bootstrap.h" +#include "env/ob_multi_replica_test_base.h" +#include "env/ob_multi_replica_util.h" +#include "lib/mysqlclient/ob_mysql_result.h" +#include "storage/tx/ob_trans_ctx_mgr_v4.h" + +#define CUR_TEST_CASE_NAME ObTxLsState + +DEFINE_MULTI_ZONE_TEST_CASE_CLASS + +MULTI_REPLICA_TEST_MAIN_FUNCTION(test_tx_ls_state_switch_); + +namespace oceanbase +{ + +static bool errsim_switch_follower_ = false; +static bool errsim_apply_SWL_ = false; +static bool block_start_working_submitting_ = false; +static share::ObLSID errsim_dup_ls_id_; +static share::ObLSID errsim_normal_ls_id_; + +namespace transaction +{ + +OB_NOINLINE int ObLSTxCtxMgr::errsim_switch_to_followr_gracefully() +{ + int ret = OB_SUCCESS; + if (errsim_switch_follower_) { + ret = OB_TIMEOUT; + } + + if (OB_FAIL(ret)) { + TRANS_LOG(INFO, "errsim in switch_to_follower_gracefully", K(ret), K(errsim_switch_follower_), + K(errsim_normal_ls_id_), K(errsim_dup_ls_id_), KPC(this)); + } + return ret; +} + +OB_NOINLINE int ObLSTxCtxMgr::errsim_submit_start_working_log() +{ + int ret = OB_SUCCESS; + + TRANS_LOG(WARN, "[ObMultiReplicaTestBase] errsim for submit_start_working_log", K(ret), + KPC(this)); + + while (block_start_working_submitting_) { + usleep(1000 * 1000); + + TRANS_LOG(WARN, "[ObMultiReplicaTestBase] errsim for submit_start_working_log", K(ret), + K(block_start_working_submitting_), KPC(this)); + } + + return ret; +} + +OB_NOINLINE int ObLSTxCtxMgr::errsim_apply_start_working_log() +{ + int ret = OB_SUCCESS; + + if (errsim_apply_SWL_) { + ret = OB_TIMEOUT; + } + if (OB_FAIL(ret)) { + TRANS_LOG(INFO, "errsim in on_start_working_log_cb_succ", K(ret), K(errsim_apply_SWL_), + K(errsim_normal_ls_id_), K(errsim_dup_ls_id_), KPC(this)); + } + + return ret; +} + +} // namespace transaction + +namespace unittest +{ + +using namespace oceanbase::transaction; +using namespace oceanbase::storage; + +struct TableBasicArg +{ + uint64_t tenant_id_; + + int64_t dup_ls_id_num_; + int64_t dup_table_id_; + ObSEArray dup_tablet_id_array_; + + int64_t normal_ls_id_num_; + int64_t normal_table_id_; + ObSEArray normal_tablet_id_array_; + + TO_STRING_KV(K(tenant_id_), + K(dup_ls_id_num_), + K(dup_table_id_), + K(normal_ls_id_num_), + K(normal_table_id_), + K(dup_tablet_id_array_), + K(normal_tablet_id_array_)); + + OB_UNIS_VERSION(1); +}; + +OB_SERIALIZE_MEMBER(TableBasicArg, + tenant_id_, + dup_ls_id_num_, + dup_table_id_, + dup_tablet_id_array_, + normal_ls_id_num_, + normal_table_id_, + normal_tablet_id_array_); + +static TableBasicArg static_basic_arg_; + +TEST_F(GET_ZONE_TEST_CLASS_NAME(1), create_test_env) +{ + int ret = OB_SUCCESS; + + const std::string test_dup_table_name = "test_dup_1"; + const std::string test_normal_table_name = "test_normal_1"; + + CREATE_TEST_TENANT(test_tenant_id); + SERVER_LOG(INFO, "[ObMultiReplicaTestBase] create test tenant success", K(test_tenant_id)); + + common::ObMySQLProxy &test_tenant_sql_proxy = get_curr_simple_server().get_sql_proxy2(); + + ACQUIRE_CONN_FROM_SQL_PROXY(test_conn, test_tenant_sql_proxy); + + std::string primary_zone_sql = "ALTER TENANT " + std::string(DEFAULT_TEST_TENANT_NAME) + + " set primary_zone='zone1; zone3; zone2';"; + WRITE_SQL_BY_CONN(test_conn, primary_zone_sql.c_str()); + + unittest::TestEnvTool::create_table_for_test_env( + test_conn, test_dup_table_name.c_str(), 10, true /*is_dup_table*/, + static_basic_arg_.dup_ls_id_num_, static_basic_arg_.dup_table_id_, + static_basic_arg_.dup_tablet_id_array_); + + unittest::TestEnvTool::create_table_for_test_env( + test_conn, test_normal_table_name.c_str(), 10, false /*is_dup_table*/, + static_basic_arg_.normal_ls_id_num_, static_basic_arg_.normal_table_id_, + static_basic_arg_.normal_tablet_id_array_); + + GET_LS(test_tenant_id, static_basic_arg_.dup_ls_id_num_, ls_handle); + SERVER_LOG(INFO, "[ObMultiReplicaTestBase] -------- before wait dup tablet discover", K(ret), + K(static_basic_arg_)); + RETRY_UNTIL_TIMEOUT(ls_handle.get_ls()->dup_table_ls_handler_.get_dup_tablet_count() + == static_basic_arg_.dup_tablet_id_array_.count(), + 20 * 1000 * 1000, 100 * 1000); + RETRY_UNTIL_TIMEOUT( + ls_handle.get_ls()->dup_table_ls_handler_.tablets_mgr_ptr_->get_readable_tablet_set_count() + >= 1, + 20 * 1000 * 1000, 100 * 1000); + RETRY_UNTIL_TIMEOUT( + ls_handle.get_ls() + ->dup_table_ls_handler_.tablets_mgr_ptr_->get_need_confirm_tablet_set_count() + == 0, + 20 * 1000 * 1000, 100 * 1000); + SERVER_LOG(INFO, "[ObMultiReplicaTestBase] -------- after wait dup tablet discover", K(ret), + K(static_basic_arg_), + K(ls_handle.get_ls()->dup_table_ls_handler_.get_dup_tablet_count())); + ASSERT_EQ(OB_SUCCESS, ret /*has_dup_tablet*/); + + WRITE_SQL_BY_CONN(test_conn, "set autocommit = false;"); + WRITE_SQL_BY_CONN(test_conn, "begin;"); + + const int64_t DEFAULT_LOAD_ROW_CNT = 10; + for (int i = 1; i <= DEFAULT_LOAD_ROW_CNT; i++) { + std::string insert_dup_sql_str = + "INSERT INTO " + test_dup_table_name + " VALUES(" + std::to_string(i) + ", 0 , 0)"; + std::string insert_normal_sql_str = + "INSERT INTO " + test_normal_table_name + " VALUES(" + std::to_string(i) + ", 0 , 0)"; + WRITE_SQL_BY_CONN(test_conn, insert_dup_sql_str.c_str()); + WRITE_SQL_BY_CONN(test_conn, insert_normal_sql_str.c_str()); + } + WRITE_SQL_BY_CONN(test_conn, "commit;"); + + errsim_apply_SWL_ = true; + + static_basic_arg_.tenant_id_ = test_tenant_id; + std::string tmp_str; + ASSERT_EQ(OB_SUCCESS, EventArgSerTool::serialize_arg(static_basic_arg_, tmp_str)); + finish_event("CREATE_TEST_TABLE", tmp_str); +} + +void switch_leader_and_check(sqlclient::ObISQLConnection *test_conn, + const int64_t tenant_id, + const int64_t ls_id_num, + const std::string local_ip, + const std::string target_ip, + const bool is_dup_ls = false) +{ + int ret = OB_SUCCESS; + + TRANS_LOG(INFO, "[ObMultiReplicaTestBase] Start switching leader to local server", K(ret), + K(tenant_id), K(ls_id_num), K(local_ip.c_str()), K(target_ip.c_str()), K(is_dup_ls)); + + GET_LS(tenant_id, ls_id_num, ls_handle); + if (local_ip == target_ip) { + if (is_dup_ls) { + RETRY_UNTIL_TIMEOUT(!ls_handle.get_ls()->dup_table_ls_handler_.is_master(), 20 * 1000 * 1000, + 100 * 1000); + ASSERT_EQ(OB_SUCCESS, ret); + } + RETRY_UNTIL_TIMEOUT(!ls_handle.get_ls()->ls_tx_svr_.mgr_->is_master(), 20 * 1000 * 1000, + 100 * 1000); + ASSERT_EQ(OB_SUCCESS, ret); + } + std::string ls_id_str = std::to_string(ls_id_num); + + std::string switch_leader_sql = "alter system switch replica leader ls=" + ls_id_str + " server='" + + target_ip + "' tenant='tt1';"; + + WRITE_SQL_BY_CONN(test_conn, switch_leader_sql.c_str()); + if (local_ip == target_ip) { + if (is_dup_ls) { + RETRY_UNTIL_TIMEOUT(ls_handle.get_ls()->dup_table_ls_handler_.is_master(), 20 * 1000 * 1000, + 100 * 1000); + ASSERT_EQ(OB_SUCCESS, ret); + } + RETRY_UNTIL_TIMEOUT(ls_handle.get_ls()->ls_tx_svr_.mgr_->is_master(), 20 * 1000 * 1000, + 100 * 1000); + ASSERT_EQ(OB_SUCCESS, ret); + } + + TRANS_LOG(INFO, "[ObMultiReplicaTestBase] Finish switching leader to local server", K(ret), + K(tenant_id), K(ls_id_num), K(local_ip.c_str()), K(target_ip.c_str()), K(is_dup_ls)); +} + +TEST_F(GET_ZONE_TEST_CLASS_NAME(2), switch_leader_to_zone2) +{ + std::string tmp_event_val; + ASSERT_EQ(OB_SUCCESS, wait_event_finish("CREATE_TEST_TABLE", tmp_event_val, 30 * 60 * 1000)); + ASSERT_EQ(OB_SUCCESS, + EventArgSerTool::deserialize_arg(static_basic_arg_, tmp_event_val)); + + common::ObMySQLProxy &test_tenant_sql_proxy = get_curr_simple_server().get_sql_proxy(); + ACQUIRE_CONN_FROM_SQL_PROXY(test_conn, test_tenant_sql_proxy); + + std::string target_ip = local_ip_ + ":" + std::to_string(rpc_ports_[1]); + switch_leader_and_check(test_conn, static_basic_arg_.tenant_id_, + static_basic_arg_.normal_ls_id_num_, target_ip, target_ip, false); + switch_leader_and_check(test_conn, static_basic_arg_.tenant_id_, static_basic_arg_.dup_ls_id_num_, + target_ip, target_ip, true); + + ASSERT_EQ(OB_SUCCESS, finish_event("SWITCH_LEADER_TO_ZONE2_GRACEFULLY", "")); +} + +bool check_follower_with_lock(ObLS *ls) +{ + bool is_follower = false; + if (ls->ls_tx_svr_.mgr_->rwlock_.try_wrlock()) { + is_follower = ls->ls_tx_svr_.mgr_->tx_ls_state_mgr_.is_follower(); + ls->ls_tx_svr_.mgr_->rwlock_.wrunlock(); + } + + return is_follower; +} + +TEST_F(GET_ZONE_TEST_CLASS_NAME(2), switch_follower_failed_from_zone2_with_start_working_on_failure) +{ + int ret = OB_SUCCESS; + std::string tmp_event_val; + ASSERT_EQ(OB_SUCCESS, + wait_event_finish("SWITCH_LEADER_TO_ZONE2_GRACEFULLY", tmp_event_val, 30 * 60 * 1000)); + common::ObMySQLProxy &test_tenant_sql_proxy = get_curr_simple_server().get_sql_proxy(); + ACQUIRE_CONN_FROM_SQL_PROXY(test_conn, test_tenant_sql_proxy); + + GET_LS(static_basic_arg_.tenant_id_, static_basic_arg_.dup_ls_id_num_, dup_ls_handle); + GET_LS(static_basic_arg_.tenant_id_, static_basic_arg_.normal_ls_id_num_, normal_ls_handle); + + errsim_dup_ls_id_ = share::ObLSID(static_basic_arg_.dup_ls_id_num_); + errsim_normal_ls_id_ = share::ObLSID(static_basic_arg_.normal_ls_id_num_); + + // switch to follower with timeout; + // resume leader and submit start_working; + std::string target_ip = local_ip_ + ":" + std::to_string(rpc_ports_[0]); + errsim_switch_follower_ = true; + + switch_leader_and_check(test_conn, static_basic_arg_.tenant_id_, + static_basic_arg_.normal_ls_id_num_, "", target_ip, false); + switch_leader_and_check(test_conn, static_basic_arg_.tenant_id_, static_basic_arg_.dup_ls_id_num_, + "", target_ip, true); + + share::SCN normal_applyied_SWL_scn = + normal_ls_handle.get_ls()->ls_tx_svr_.mgr_->tx_ls_state_mgr_.max_applied_start_working_ts_; + usleep(50 * 1000); + RETRY_UNTIL_TIMEOUT(check_follower_with_lock(normal_ls_handle.get_ls()), 10 * 1000 * 1000, + 100 * 1000); + TRANS_LOG(INFO, "[ObMultiReplicaTestBase] zone2 can not become a follower", K(ret), + KPC(normal_ls_handle.get_ls()->ls_tx_svr_.mgr_)); + ASSERT_EQ(ret, OB_TIMEOUT); + + ret = OB_SUCCESS; + ASSERT_EQ( + normal_ls_handle.get_ls()->ls_tx_svr_.mgr_->tx_ls_state_mgr_.max_applied_start_working_ts_ + > normal_applyied_SWL_scn, + true); + + RETRY_UNTIL_TIMEOUT(normal_ls_handle.get_ls()->ls_tx_svr_.mgr_->tx_ls_state_mgr_.is_follower(), + 30 * 1000 * 1000, 1 * 1000); + ASSERT_EQ(ret, OB_SUCCESS); + block_start_working_submitting_ = true; + + // block msg with a busy start_working_cb + ATOMIC_STORE(&block_msg_, true); + TRANS_LOG(INFO, "[ObMultiReplicaTestBase] Start to block msg", K(ret), + KPC(normal_ls_handle.get_ls()->ls_tx_svr_.mgr_)); + + finish_event("BLOCK_ZONE2_MSG_PROCESS", ""); + + ASSERT_EQ(OB_SUCCESS, + wait_event_finish("BLOCK_ZONE2_MSG_PROCESS", tmp_event_val, 30 * 60 * 1000)); + + usleep(1 * 1000 * 1000); + block_start_working_submitting_ = false; + + RETRY_UNTIL_TIMEOUT( + !normal_ls_handle.get_ls()->ls_tx_svr_.mgr_->ls_log_writer_.start_working_cbs_.is_empty(), + 5 * 1000 * 1000, 10 * 1000); + TRANS_LOG(INFO, "[ObMultiReplicaTestBase] wait a pending start_working_cb", K(ret), + KPC(normal_ls_handle.get_ls()->ls_tx_svr_.mgr_)); + ASSERT_EQ(ret, OB_SUCCESS); + ASSERT_EQ(normal_ls_handle.get_ls()->ls_tx_svr_.mgr_->is_r_pending_(), true); + // wait election lease expired + usleep(15 * 1000 * 1000); + + finish_event("ZONE1_SUBMIT_LAST_START_WORKING", ""); + + ASSERT_EQ(OB_SUCCESS, + wait_event_finish("ZONE1_BECOME_LEADER_IN_BLOCK_MSG", tmp_event_val, 30 * 60 * 1000)); + TRANS_LOG(INFO, "[ObMultiReplicaTestBase] The zone 1 has been a new leader", K(ret), + KPC(normal_ls_handle.get_ls()->ls_tx_svr_.mgr_)); + ASSERT_EQ(ret, OB_SUCCESS); + + // usleep(2 * 1000 * 1000); + + ATOMIC_STORE(&block_msg_, false); + TRANS_LOG(INFO, "[ObMultiReplicaTestBase] Finish to block msg", K(ret), + KPC(normal_ls_handle.get_ls()->ls_tx_svr_.mgr_)); + + // TODO: start_working on failure + // RETRY_UNTIL_TIMEOUT( + // normal_ls_handle.get_ls()->ls_tx_svr_.mgr_->tx_ls_state_mgr_.cur_state_.state_val_.state_ + // == ObTxLSStateMgr::TxLSState::R_SYNC_FAILED, + // 5 * 1000 * 1000, 10 * 1000); + // ASSERT_EQ(ret, OB_SUCCESS); + + RETRY_UNTIL_TIMEOUT( + normal_ls_handle.get_ls()->ls_tx_svr_.mgr_->tx_ls_state_mgr_.cur_state_.state_val_.state_ + == ObTxLSStateMgr::TxLSState::F_WORKING, + 5 * 1000 * 1000, 10 * 1000); + ASSERT_EQ(ret, OB_SUCCESS); + + errsim_switch_follower_ = false; + + finish_event("SWITCH_FOLLOWER_FAILED_FROM_ZONE2", ""); +} + +TEST_F(GET_ZONE_TEST_CLASS_NAME(1), switch_leader_to_zone1_with_start_working_error) +{ + int ret = OB_SUCCESS; + std::string tmp_event_val; + + ASSERT_EQ(OB_SUCCESS, + wait_event_finish("BLOCK_ZONE2_MSG_PROCESS", tmp_event_val, 30 * 60 * 1000)); + ATOMIC_STORE(&block_msg_, true); + + finish_event("BLOCK_ZONE1_MSG_PROCESS", ""); + + ASSERT_EQ(OB_SUCCESS, + wait_event_finish("ZONE1_SUBMIT_LAST_START_WORKING", tmp_event_val, 30 * 60 * 1000)); + ATOMIC_STORE(&block_msg_, false); + + GET_LS(static_basic_arg_.tenant_id_, static_basic_arg_.normal_ls_id_num_, normal_ls_handle); + // switch to leader and submit start_working log + RETRY_UNTIL_TIMEOUT( + normal_ls_handle.get_ls()->ls_tx_svr_.mgr_->tx_ls_state_mgr_.is_start_working_apply_pending(), + 20 * 1000 * 1000, 100 * 1000); + ASSERT_EQ(OB_SUCCESS, ret); + TRANS_LOG(INFO, "[ObMultiReplicaTestBase] The zone 1 has been a new leader", K(ret), + KPC(normal_ls_handle.get_ls()->ls_tx_svr_.mgr_)); + finish_event("ZONE1_BECOME_LEADER_IN_BLOCK_MSG", ""); + + RETRY_UNTIL_TIMEOUT(normal_ls_handle.get_ls()->ls_tx_svr_.mgr_->tx_ls_state_mgr_.is_master(), + 20 * 1000 * 1000, 1 * 1000 * 1000); + ASSERT_EQ(OB_TIMEOUT, ret); + + finish_event("KEEP_SWL_PENDING_IN_ZONE1", ""); +} + +TEST_F(GET_ZONE_TEST_CLASS_NAME(1), transfer_with_block_normal) +{ + // TODO +} + +TEST_F(GET_ZONE_TEST_CLASS_NAME(2), offline_ls_with_retry) +{ + // TODO +} + +TEST_F(GET_ZONE_TEST_CLASS_NAME(2), gc_ls) +{ + // TODO +} + +} // namespace unittest +} // namespace oceanbase diff --git a/mittest/simple_server/CMakeLists.txt b/mittest/simple_server/CMakeLists.txt index 93b7cb1935..91e9357f97 100644 --- a/mittest/simple_server/CMakeLists.txt +++ b/mittest/simple_server/CMakeLists.txt @@ -35,6 +35,7 @@ function(ob_offline_observer case case_file) EXCLUDE_FROM_ALL ${case_file} ${OBSERVER_TEST_SRCS} + ../${MIT_SRCS} ) target_include_directories(${case} PUBLIC ${CMAKE_SOURCE_DIR}/unittest ${CMAKE_SOURCE_DIR}/mittest) @@ -53,6 +54,7 @@ endfunction() ob_offline_observer(test_simple_ob test_ob_simple_cluster.cpp) ob_offline_observer(test_transfer_tx test_transfer_tx.cpp) +ob_offline_observer(test_tx_data test_tx_data.cpp) ob_unittest_observer(test_transfer_no_kill_tx test_transfer_tx.cpp) ob_unittest_observer(test_standby_balance test_standby_balance_ls_group.cpp) diff --git a/mittest/simple_server/env/ob_simple_cluster_test_base.cpp b/mittest/simple_server/env/ob_simple_cluster_test_base.cpp index a873972c0d..748543f584 100644 --- a/mittest/simple_server/env/ob_simple_cluster_test_base.cpp +++ b/mittest/simple_server/env/ob_simple_cluster_test_base.cpp @@ -106,7 +106,8 @@ ObSimpleClusterTestBase::~ObSimpleClusterTestBase() void ObSimpleClusterTestBase::SetUp() { - SERVER_LOG(INFO, "SetUp"); + auto case_name = ::testing::UnitTest::GetInstance()->current_test_info()->name(); + SERVER_LOG(INFO, "SetUp>>>>>>>>>>>>>>", K(case_name)); int ret = OB_SUCCESS; if (!is_started_) { if (OB_FAIL(start())) { @@ -126,7 +127,8 @@ void ObSimpleClusterTestBase::SetUp() void ObSimpleClusterTestBase::TearDown() { - + auto case_name = ::testing::UnitTest::GetInstance()->current_test_info()->name(); + SERVER_LOG(INFO, "TearDown>>>>>>>>>>>>>>", K(case_name)); } void ObSimpleClusterTestBase::TearDownTestCase() diff --git a/mittest/simple_server/rewrite_function_for_test_big_tx_data.cpp b/mittest/simple_server/rewrite_function_for_test_big_tx_data.cpp index 084594d054..1e5be9d36a 100644 --- a/mittest/simple_server/rewrite_function_for_test_big_tx_data.cpp +++ b/mittest/simple_server/rewrite_function_for_test_big_tx_data.cpp @@ -84,9 +84,10 @@ int ObTxData::add_undo_action(ObTxTable *tx_table, // STORAGE_LOG(DEBUG, "do add_undo_action"); UNUSED(undo_node); int ret = OB_SUCCESS; - SpinWLockGuard guard(undo_status_list_.lock_); + init_tx_op(); + SpinWLockGuard guard(op_guard_->get_undo_status_list().lock_); ObTxDataTable *tx_data_table = nullptr; - ObUndoStatusNode *node = undo_status_list_.head_; + ObUndoStatusNode *node = op_guard_->get_undo_status_list().head_; if (OB_ISNULL(tx_table)) { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "tx table is nullptr.", KR(ret)); @@ -104,9 +105,9 @@ int ObTxData::add_undo_action(ObTxTable *tx_table, STORAGE_LOG(WARN, "alloc_undo_status_node() fail", KR(ret)); } else { new_node->next_ = node; - undo_status_list_.head_ = new_node; + op_guard_->get_undo_status_list().head_ = new_node; node = new_node; - undo_status_list_.undo_node_cnt_++; + op_guard_->get_undo_status_list().undo_node_cnt_++; } for (int64_t idx = 0; idx < TX_DATA_UNDO_ACT_MAX_NUM_PER_NODE; ++idx) { node->undo_actions_[node->size_++] = new_undo_action; @@ -140,8 +141,8 @@ int ObTxDataMemtableScanIterator // not exactly accurate, but enough for unittest ATOMIC_STORE(&BIGGEST_TX_DATA_SIZE, buffer_len_); } - if (tx_data_->undo_status_list_.undo_node_cnt_ > 0) { - std::cout << "tx_id:" << tx_data_->tx_id_.get_id() << ", undo cnt:" << tx_data_->undo_status_list_.undo_node_cnt_ << ", generate size:" << generate_size_ << std::endl; + if (tx_data_->op_guard_->get_undo_status_list().undo_node_cnt_ > 0) { + std::cout << "tx_id:" << tx_data_->tx_id_.get_id() << ", undo cnt:" << tx_data_->op_guard_->get_undo_status_list().undo_node_cnt_ << ", generate size:" << generate_size_ << std::endl; } ATOMIC_STORE(&DUMP_BIG_TX_DATA, true); /**************************************************************************************************/ @@ -225,7 +226,11 @@ int ObTxDataSingleRowGetter::deserialize_tx_data_from_store_buffers_(ObTxData &t tx_data.tx_id_.get_id(), tx_data_buffers_.count()); ATOMIC_STORE(&LOAD_BIG_TX_DATA, true); - std::cout << "read big tx id from sstable, tx_id:" << ATOMIC_LOAD(&TEST_TX_ID) << ", undo cnt:" << tx_data.undo_status_list_.undo_node_cnt_ << ", buffer cnt:" << tx_data_buffers_.count() << std::endl; + int64_t undo_cnt = 0; + if (tx_data.op_guard_.is_valid()) { + undo_cnt = tx_data.op_guard_->get_undo_status_list().undo_node_cnt_; + } + std::cout << "read big tx id from sstable, tx_id:" << ATOMIC_LOAD(&TEST_TX_ID) << ", undo cnt:" << undo_cnt << ", buffer cnt:" << tx_data_buffers_.count() << std::endl; } } /**************************************************************************************************/ diff --git a/mittest/simple_server/test_big_tx_data.cpp b/mittest/simple_server/test_big_tx_data.cpp index 996d07c04b..94529d132f 100644 --- a/mittest/simple_server/test_big_tx_data.cpp +++ b/mittest/simple_server/test_big_tx_data.cpp @@ -64,7 +64,7 @@ class DoNothingOP : public ObITxDataCheckFunctor { virtual int operator()(const ObTxData &tx_data, ObTxCCCtx *tx_cc_ctx = nullptr) { UNUSED(tx_cc_ctx); - cout << "read tx data:" << tx_data.tx_id_.get_id() << ", undo cnt:" << tx_data.undo_status_list_.undo_node_cnt_ << endl; + cout << "read tx data:" << tx_data.tx_id_.get_id() << ", undo cnt:" << tx_data.op_guard_->get_undo_status_list().undo_node_cnt_ << endl; STORAGE_LOG_RET(INFO, 0, "read tx data", K(tx_data.tx_id_), K(lbt())); return OB_SUCCESS; } diff --git a/mittest/simple_server/test_transfer_tx.cpp b/mittest/simple_server/test_transfer_tx.cpp index 830653709a..e30a7133b6 100644 --- a/mittest/simple_server/test_transfer_tx.cpp +++ b/mittest/simple_server/test_transfer_tx.cpp @@ -80,6 +80,8 @@ int ObSimpleClusterExampleTest::do_balance_inner_(uint64_t tenant_id) ObBalanceJob job; if (OB_FAIL(b_svr->gather_stat_())) { LOG_WARN("failed to gather stat", KR(ret)); + } else if (OB_FAIL(b_svr->gather_ls_status_stat(tenant_id, b_svr->ls_array_))) { + LOG_WARN("failed to gather stat", KR(ret)); } else if (OB_FAIL(ObBalanceJobTableOperator::get_balance_job( tenant_id, false, *GCTX.sql_proxy_, job, start_time, finish_time))) { if (OB_ENTRY_NOT_EXIST == ret) { @@ -352,6 +354,7 @@ TEST_F(ObSimpleClusterExampleTest, tx_exit) EQ(OB_TRANS_CTX_NOT_EXIST, SSH::wait_tx_exit(R.tenant_id_, loc2, tx_id)); } +/* TEST_F(ObSimpleClusterExampleTest, large_query) { TRANSFER_CASE_PREPARE; @@ -435,7 +438,7 @@ TEST_F(ObSimpleClusterExampleTest, large_query) LOGI("large_query: row_count:%ld", row_count); //get_curr_simple_server().get_sql_proxy().write("alter system set syslog_level='INFO'", affected_rows); } - +*/ TEST_F(ObSimpleClusterExampleTest, epoch_recover_from_active_info) { @@ -1215,7 +1218,9 @@ TEST_F(ObSimpleClusterExampleTest, transfer_tx_ctx_merge) TEST_F(ObSimpleClusterExampleTest, transfer_batch) { TRANSFER_CASE_PREPARE; + sql_proxy.write("alter system set _transfer_start_trans_timeout='5s'",affected_rows); + sql_proxy.write("alter system set _transfer_start_trans_timeout = '10s'", affected_rows); std::set jobs; for (int i =0 ;i< 5000;i++) { sqlclient::ObISQLConnection *conn = NULL; @@ -1243,6 +1248,7 @@ TEST_F(ObSimpleClusterExampleTest, transfer_batch) int64_t sum = 0; EQ(0, SSH::select_int64(sql_proxy, "select sum(col) as val from stu2", sum)); EQ(100 * 5000, sum); + sql_proxy.write("alter system set _transfer_start_trans_timeout='1s'",affected_rows); } TEST_F(ObSimpleClusterExampleTest, transfer_retain_ctx) diff --git a/mittest/simple_server/test_transfer_with_smaller_tx_data.cpp b/mittest/simple_server/test_transfer_with_smaller_tx_data.cpp index 80f95df714..46eb1bb456 100644 --- a/mittest/simple_server/test_transfer_with_smaller_tx_data.cpp +++ b/mittest/simple_server/test_transfer_with_smaller_tx_data.cpp @@ -320,7 +320,7 @@ TEST_F(ObTransferWithSmallerStartSCN, smaller_start_scn) ASSERT_EQ(0, SSH::submit_redo(tenant_id, loc1)); ObTxLoopWorker *worker = MTL(ObTxLoopWorker *); - worker->scan_all_ls_(true, true); + worker->scan_all_ls_(true, true, false); usleep(1 * 1000 * 1000); // Step4: let the tx data table update upper info @@ -397,7 +397,7 @@ TEST_F(ObTransferWithSmallerStartSCN, smaller_start_scn) } ASSERT_EQ(loc1, loc2); - worker->scan_all_ls_(true, true); + worker->scan_all_ls_(true, true, false); usleep(1 * 1000 * 1000); fprintf(stdout, "start update upper info the second time\n"); diff --git a/mittest/simple_server/test_tx_ctx_table_mit.cpp b/mittest/simple_server/test_tx_ctx_table_mit.cpp index 072b96a2dd..3f4ea07de0 100644 --- a/mittest/simple_server/test_tx_ctx_table_mit.cpp +++ b/mittest/simple_server/test_tx_ctx_table_mit.cpp @@ -85,7 +85,7 @@ int ObTxCtxMemtableScanIterator::serialize_next_tx_ctx_(ObTxLocalBuffer &buffer, if (OB_FAIL(ret)) { STORAGE_LOG(INFO, "get next tx ctx table info failed", KR(ret), KPC(tx_ctx)); } else if (SLEEP_BEFORE_DUMP_TX_CTX) { - fprintf(stdout, "ready to dump tx ctx, undo status node ptr : %p\n", tx_ctx->ctx_tx_data_.tx_data_guard_.tx_data()->undo_status_list_.head_); + fprintf(stdout, "ready to dump tx ctx, undo status node ptr : %p\n", tx_ctx->ctx_tx_data_.tx_data_guard_.tx_data()->op_guard_->get_undo_status_list().head_); fprintf(stdout, "sleep 20 seconds before dump\n"); HAS_GOT_TX_CTX = true; SLEEP_BEFORE_DUMP_TX_CTX = false; diff --git a/mittest/simple_server/test_tx_data.cpp b/mittest/simple_server/test_tx_data.cpp new file mode 100644 index 0000000000..39961fe0fe --- /dev/null +++ b/mittest/simple_server/test_tx_data.cpp @@ -0,0 +1,373 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#define USING_LOG_PREFIX SERVER +#define protected public +#define private public + +#include "env/ob_simple_cluster_test_base.h" +#include "mittest/env/ob_simple_server_helper.h" +#include "storage/tx_storage/ob_ls_service.h" + +namespace oceanbase +{ +namespace unittest +{ + +using namespace oceanbase::transaction; +using namespace oceanbase::storage; + + +#define EQ(x, y) GTEST_ASSERT_EQ(x, y); +#define NEQ(x, y) GTEST_ASSERT_NE(x, y); +#define LE(x, y) GTEST_ASSERT_LE(x, y); +#define GE(x, y) GTEST_ASSERT_GE(x, y); + +class TestRunCtx +{ +public: + uint64_t tenant_id_ = 0; + int64_t time_sec_ = 0; +}; + +TestRunCtx R; + +class ObTxDataTest : public ObSimpleClusterTestBase +{ +public: + // 指定case运行目录前缀 test_ob_simple_cluster_ + ObTxDataTest() : ObSimpleClusterTestBase("test_tx_data_", "50G", "50G") {} +}; + +TEST_F(ObTxDataTest, observer_start) +{ + SERVER_LOG(INFO, "observer_start succ"); +} + +// 创建租户并不轻量，看场景必要性使用 +TEST_F(ObTxDataTest, add_tenant) +{ + // 创建普通租户tt1 + ASSERT_EQ(OB_SUCCESS, create_tenant("tt1", "40G", "40G", false, 10)); + // 获取租户tt1的tenant_id + ASSERT_EQ(OB_SUCCESS, get_tenant_id(R.tenant_id_)); + ASSERT_NE(0, R.tenant_id_); + // 初始化普通租户tt1的sql proxy + ASSERT_EQ(OB_SUCCESS, get_curr_simple_server().init_sql_proxy2()); +} + +TEST_F(ObTxDataTest, create_new_ls) +{ + // 在单节点ObServer下创建新的日志流, 注意避免被RS任务GC掉 + EQ(0, SSH::create_ls(R.tenant_id_, get_curr_observer().self_addr_)); + int64_t ls_count = 0; + EQ(0, SSH::g_select_int64(R.tenant_id_, "select count(ls_id) as val from __all_ls where ls_id!=1", ls_count)); + EQ(2, ls_count); +} + +TEST_F(ObTxDataTest, rollback_to) +{ + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); + int64_t affected_rows; + EQ(0, sql_proxy.write("drop table if exists stu1", affected_rows)); + EQ(0, sql_proxy.write("create table stu1(col1 int)", affected_rows)); + sqlclient::ObISQLConnection *conn1 = NULL; + EQ(0, sql_proxy.acquire(conn1)); + EQ(0, SSH::write(conn1, "set autocommit=0", affected_rows)); + EQ(0, SSH::write(conn1, "insert into stu1 values(100)", affected_rows)); + ObTransID tx_id; + EQ(0, SSH::find_tx(conn1, tx_id)); + LOGI("find_tx:%ld", tx_id.get_id()); + EQ(0, SSH::write(conn1, "savepoint sp1")); + EQ(0, SSH::write(conn1, "insert into stu1 values(200)", affected_rows)); + EQ(0, SSH::write(conn1, "rollback to sp1")); + int64_t val = 0; + EQ(0, SSH::select_int64(conn1, "select sum(col1) val from stu1", val)); + EQ(100, val); + EQ(0, SSH::write(conn1, "commit")); +} + +TEST_F(ObTxDataTest, rollback_to_with_redo) +{ + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); + int64_t affected_rows; + EQ(0, sql_proxy.write("drop table if exists stu1", affected_rows)); + EQ(0, sql_proxy.write("create table stu1(col1 int)", affected_rows)); + + sqlclient::ObISQLConnection *conn1 = NULL; + EQ(0, sql_proxy.acquire(conn1)); + EQ(0, SSH::write(conn1, "set autocommit=0", affected_rows)); + EQ(0, SSH::write(conn1, "insert into stu1 values(100)", affected_rows)); + ObTransID tx_id; + EQ(0, SSH::find_tx(conn1, tx_id)); + LOGI("find_tx:%ld", tx_id.get_id()); + + EQ(0, SSH::write(conn1, "savepoint sp1")); + EQ(0, SSH::write(conn1, "insert into stu1 values(200)", affected_rows)); + ObLSID loc1; + EQ(0, SSH::select_table_loc(R.tenant_id_, "stu1", loc1)); + // when tx has redo, rollback to need write log + EQ(0, SSH::submit_redo(R.tenant_id_, loc1)); + + EQ(0, SSH::write(conn1, "rollback to sp1")); + int64_t val = 0; + EQ(0, SSH::select_int64(conn1, "select sum(col1) val from stu1", val)); + EQ(100, val); + EQ(0, SSH::write(conn1, "commit")); +} + +TEST_F(ObTxDataTest, rollback_to_with_read_sstable_uncommit) +{ + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); + int64_t affected_rows; + EQ(0, sql_proxy.write("drop table if exists stu1", affected_rows)); + EQ(0, sql_proxy.write("create table stu1(col1 int)", affected_rows)); + + sqlclient::ObISQLConnection *conn1 = NULL; + EQ(0, sql_proxy.acquire(conn1)); + EQ(0, SSH::write(conn1, "set autocommit=0", affected_rows)); + EQ(0, SSH::write(conn1, "insert into stu1 values(100)", affected_rows)); + ObTransID tx_id; + EQ(0, SSH::find_tx(conn1, tx_id)); + LOGI("find_tx:%ld", tx_id.get_id()); + + EQ(0, SSH::write(conn1, "savepoint sp1")); + EQ(0, SSH::write(conn1, "insert into stu1 values(200)", affected_rows)); + ObLSID loc1; + EQ(0, SSH::select_table_loc(R.tenant_id_, "stu1", loc1)); + // when tx has redo, rollback to need write log + EQ(0, SSH::submit_redo(R.tenant_id_, loc1)); + + EQ(0, SSH::write(conn1, "rollback to sp1")); + int64_t val = 0; + EQ(0, SSH::select_int64(conn1, "select sum(col1) val from stu1", val)); + EQ(100, val); + + EQ(0, sql_proxy.write("alter system minor freeze", affected_rows)); + EQ(0, SSH::wait_checkpoint_newest(R.tenant_id_, loc1)); + // read from sstable uncommit row + EQ(0, SSH::select_int64(conn1, "select sum(col1) val from stu1", val)); + EQ(100, val); + + EQ(0, SSH::write(conn1, "commit")); + EQ(0, SSH::select_int64(sql_proxy, "select sum(col1) as val from stu1",val)); + EQ(100, val); +} + +TEST_F(ObTxDataTest, rollback_to_with_ls_replay) +{ + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); + int64_t affected_rows; + EQ(0, sql_proxy.write("drop table if exists stu1", affected_rows)); + EQ(0, sql_proxy.write("create table stu1(col1 int)", affected_rows)); + + sqlclient::ObISQLConnection *conn1 = NULL; + EQ(0, sql_proxy.acquire(conn1)); + EQ(0, SSH::write(conn1, "set autocommit=0", affected_rows)); + EQ(0, SSH::write(conn1, "insert into stu1 values(100)", affected_rows)); + ObTransID tx_id; + EQ(0, SSH::find_tx(conn1, tx_id)); + LOGI("find_tx:%ld", tx_id.get_id()); + + EQ(0, SSH::write(conn1, "savepoint sp1")); + EQ(0, SSH::write(conn1, "insert into stu1 values(200)", affected_rows)); + ObLSID loc1; + EQ(0, SSH::select_table_loc(R.tenant_id_, "stu1", loc1)); + // when tx has redo, rollback to need write log + EQ(0, SSH::submit_redo(R.tenant_id_, loc1)); + + EQ(0, SSH::write(conn1, "rollback to sp1")); + int64_t val = 0; + EQ(0, SSH::select_int64(conn1, "select sum(col1) val from stu1", val)); + EQ(100, val); + + EQ(0, sql_proxy.write("alter system minor freeze", affected_rows)); + EQ(0, SSH::wait_checkpoint_newest(R.tenant_id_, loc1)); + // read from sstable uncommit row + EQ(0, SSH::select_int64(conn1, "select sum(col1) val from stu1", val)); + EQ(100, val); + + LOGI("ls_reboot:%ld", loc1.id()); + // tx has not commit, tx ctx recover from tx_sstable + EQ(0, SSH::ls_reboot(R.tenant_id_, loc1)); + + EQ(0, SSH::write(conn1, "commit")); + EQ(0, SSH::select_int64(sql_proxy, "select sum(col1) as val from stu1",val)); + EQ(100, val); +} + +TEST_F(ObTxDataTest, rollback_to_with_ls_replay_from_middle) +{ + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); + int64_t affected_rows; + EQ(0, sql_proxy.write("drop table if exists stu1", affected_rows)); + EQ(0, sql_proxy.write("create table stu1(col1 int)", affected_rows)); + + sqlclient::ObISQLConnection *conn1 = NULL; + EQ(0, sql_proxy.acquire(conn1)); + EQ(0, SSH::write(conn1, "set autocommit=0", affected_rows)); + EQ(0, SSH::write(conn1, "insert into stu1 values(100)", affected_rows)); + ObTransID tx_id; + EQ(0, SSH::find_tx(conn1, tx_id)); + LOGI("find_tx:%ld", tx_id.get_id()); + + EQ(0, SSH::write(conn1, "savepoint sp1")); + EQ(0, SSH::write(conn1, "insert into stu1 values(200)", affected_rows)); + ObLSID loc1; + EQ(0, SSH::select_table_loc(R.tenant_id_, "stu1", loc1)); + // when tx has redo, rollback to need write log + EQ(0, SSH::submit_redo(R.tenant_id_, loc1)); + + EQ(0, sql_proxy.write("alter system minor freeze", affected_rows)); + EQ(0, SSH::wait_checkpoint_newest(R.tenant_id_, loc1)); + + EQ(0, SSH::write(conn1, "rollback to sp1")); + int64_t val = 0; + EQ(0, SSH::select_int64(conn1, "select sum(col1) val from stu1", val)); + EQ(100, val); + + EQ(0, SSH::write(conn1, "commit")); + // make tx_ctx checkpoint + EQ(0, SSH::freeze_tx_ctx(R.tenant_id_, loc1)); + + LOGI("ls_reboot:%ld", loc1.id()); + EQ(0, SSH::ls_reboot(R.tenant_id_, loc1)); + + EQ(0, SSH::select_int64(sql_proxy, "select sum(col1) as val from stu1",val)); + EQ(100, val); +} + +TEST_F(ObTxDataTest, retain_ctx) +{ + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); + int64_t affected_rows = 0; + ObMySQLTransaction trans; + EQ(0, trans.start(GCTX.sql_proxy_, R.tenant_id_)); + observer::ObInnerSQLConnection *conn = static_cast(trans.get_connection()); + char buf[10]; + ObRegisterMdsFlag flag; + ObLSID ls_id1(1001); + ObLSID ls_id2(1002); + EQ(0, conn->register_multi_data_source(R.tenant_id_, + ls_id1, + ObTxDataSourceType::TEST3, + buf, + 10, + flag)); + + EQ(0, SSH::submit_redo(R.tenant_id_, ls_id1)); + EQ(0, sql_proxy.write("alter system minor freeze", affected_rows)); + EQ(0, SSH::wait_checkpoint_newest(R.tenant_id_, ls_id1)); + + EQ(0, conn->register_multi_data_source(R.tenant_id_, + ls_id2, + ObTxDataSourceType::TEST3, + buf, + 10, + flag)); + ObTransID tx_id; + EQ(0, SSH::g_select_int64(R.tenant_id_, "select trans_id as val from __all_virtual_trans_stat where is_exiting=0 and session_id<=1 limit 1", tx_id.tx_id_)); + LOGI("find active_tx tx_id:%ld", tx_id.get_id()); + + EQ(0, trans.end(true)); + // make tx_ctx checkpoint + EQ(0, SSH::freeze_tx_ctx(R.tenant_id_, ls_id1)); + LOGI("ls_reboot:%ld", ls_id1.id()); + EQ(0, SSH::ls_reboot(R.tenant_id_, ls_id1)); + + EQ(0, SSH::freeze_tx_ctx(R.tenant_id_, ls_id2)); + LOGI("ls_reboot:%ld", ls_id2.id()); + EQ(0, SSH::ls_reboot(R.tenant_id_, ls_id2)); +} + +TEST_F(ObTxDataTest, retain_ctx2) +{ + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); + int64_t affected_rows = 0; + EQ(0, sql_proxy.write("drop table if exists stu1", affected_rows)); + EQ(0, sql_proxy.write("create table stu1(col1 int)", affected_rows)); + ObMySQLTransaction trans; + EQ(0, trans.start(GCTX.sql_proxy_, R.tenant_id_)); + observer::ObInnerSQLConnection *conn = static_cast(trans.get_connection()); + char buf[10]; + ObRegisterMdsFlag flag; + ObLSID ls_id1(1001); + EQ(0, conn->register_multi_data_source(R.tenant_id_, + ls_id1, + ObTxDataSourceType::TEST3, + buf, + 10, + flag)); + + EQ(0, SSH::submit_redo(R.tenant_id_, ls_id1)); + EQ(0, sql_proxy.write("alter system minor freeze", affected_rows)); + EQ(0, SSH::wait_checkpoint_newest(R.tenant_id_, ls_id1)); + + EQ(0, sql_proxy.write("insert into stu1 values(100)", affected_rows)); + + EQ(0, conn->register_multi_data_source(R.tenant_id_, + ls_id1, + ObTxDataSourceType::TEST3, + buf, + 10, + flag)); + ObTransID tx_id; + EQ(0, SSH::g_select_int64(R.tenant_id_, "select trans_id as val from __all_virtual_trans_stat where is_exiting=0 and session_id<=1 limit 1", tx_id.tx_id_)); + LOGI("find active_tx tx_id:%ld", tx_id.get_id()); + + EQ(0, trans.end(true)); + // make tx_ctx checkpoint + EQ(0, SSH::freeze_tx_ctx(R.tenant_id_, ls_id1)); + // make tx_data checkpoint + EQ(0, SSH::freeze_tx_data(R.tenant_id_, ls_id1)); + LOGI("ls_reboot:%ld", ls_id1.id()); + EQ(0, SSH::ls_reboot(R.tenant_id_, ls_id1)); +} + +TEST_F(ObTxDataTest, end) +{ + if (R.time_sec_ > 0) { + ::sleep(R.time_sec_); + } +} + +} // end unittest +} // end oceanbase + + +int main(int argc, char **argv) +{ + int64_t c = 0; + int64_t time_sec = 0; + char *log_level = (char*)"INFO"; + while(EOF != (c = getopt(argc,argv,"t:l:"))) { + switch(c) { + case 't': + time_sec = atoi(optarg); + break; + case 'l': + log_level = optarg; + oceanbase::unittest::ObSimpleClusterTestBase::enable_env_warn_log_ = false; + break; + default: + break; + } + } + oceanbase::unittest::init_log_and_gtest(argc, argv); + OB_LOGGER.set_log_level(log_level); + + LOG_INFO("main>>>"); + oceanbase::unittest::R.time_sec_ = time_sec; + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/src/logservice/libobcdc/src/ob_log_ls_fetch_ctx.cpp b/src/logservice/libobcdc/src/ob_log_ls_fetch_ctx.cpp index 5b63c80af7..3674ca49b4 100644 --- a/src/logservice/libobcdc/src/ob_log_ls_fetch_ctx.cpp +++ b/src/logservice/libobcdc/src/ob_log_ls_fetch_ctx.cpp @@ -151,7 +151,7 @@ int LSFetchCtx::init( const int64_t start_tstamp_ns = start_parameters.get_start_tstamp_ns(); const palf::LSN &start_lsn = start_parameters.get_start_lsn(); // If the start lsn is 0, the service is started from creation - const bool start_serve_from_create = (palf::PALF_INITIAL_LSN_VAL == start_lsn.val_); + const bool start_serve_from_create = !tls_id.is_sys_log_stream() && palf::PALF_INITIAL_LSN_VAL == start_lsn.val_; ObBackupDest archive_dest; reset(); diff --git a/src/logservice/libobcdc/src/ob_log_meta_manager.cpp b/src/logservice/libobcdc/src/ob_log_meta_manager.cpp index f835b864d5..91799e9be9 100644 --- a/src/logservice/libobcdc/src/ob_log_meta_manager.cpp +++ b/src/logservice/libobcdc/src/ob_log_meta_manager.cpp @@ -248,8 +248,10 @@ int ObLogMetaManager::get_table_meta( ret = OB_ERR_UNEXPECTED; LOG_ERROR("expect valid schema_getter", KR(ret)); } else { + ObTimeGuard time_guard("get_table_meta", 2 * 1000 * 1000); RETRY_FUNC(stop_flag, *schema_getter, get_schema_guard_and_full_table_schema, tenant_id, table_id, global_schema_version, GET_SCHEMA_TIMEOUT, - schema_mgr, table_schema); + schema_mgr, table_schema); + time_guard.click("get_full_table_schema"); if (OB_FAIL(ret)) { // caller deal with error code OB_TENANT_HAS_BEEN_DROPPED diff --git a/src/logservice/libobcdc/src/ob_log_rocksdb_store_service.cpp b/src/logservice/libobcdc/src/ob_log_rocksdb_store_service.cpp index 07e7a0b73d..d8d08d8026 100644 --- a/src/logservice/libobcdc/src/ob_log_rocksdb_store_service.cpp +++ b/src/logservice/libobcdc/src/ob_log_rocksdb_store_service.cpp @@ -26,6 +26,36 @@ #include "rocksdb/table_properties.h" #include "rocksdb/utilities/table_properties_collectors.h" +#define RETRY_FUNC_ON_IO_ERROR_WITH_USLEEP_MS(stop_flag, sleep_ms, var, func, args...) \ + do {\ + if (OB_SUCC(ret)) \ + { \ + int64_t _retry_func_on_error_last_print_time = common::ObClockGenerator::getClock();\ + int64_t _retry_func_on_error_cur_print_time = 0;\ + const int64_t _PRINT_RETRY_FUNC_INTERVAL = 10 * _SEC_;\ + s = rocksdb::Status::IOError();\ + while (s.IsIOError() && ! (stop_flag)) \ + { \ + s = (var).func(args); \ + if (s.IsIOError()) { \ + ob_usleep(sleep_ms); \ + }\ + _retry_func_on_error_cur_print_time = common::ObClockGenerator::getClock();\ + if (_retry_func_on_error_cur_print_time - _retry_func_on_error_last_print_time >= _PRINT_RETRY_FUNC_INTERVAL) {\ + LOG_DBA_WARN(OB_IO_ERROR, \ + "msg", "put value into rocksdb failed", \ + "error", s.ToString().c_str(), \ + "last_print_time", _retry_func_on_error_last_print_time); \ + _retry_func_on_error_last_print_time = _retry_func_on_error_cur_print_time; \ + }\ + } \ + if ((stop_flag)) \ + { \ + ret = OB_IN_STOP_STATE; \ + } \ + } \ + } while (0) + namespace oceanbase { namespace libobcdc @@ -176,8 +206,8 @@ int RocksDbStoreService::put(const std::string &key, const ObSlice &value) ret = OB_IN_STOP_STATE; } else { // find column family handle for cf - rocksdb::Status s = m_db_->Put( - writer_options, + rocksdb::Status s; + RETRY_FUNC_ON_IO_ERROR_WITH_USLEEP_MS(is_stopped(), 1 * _SEC_, (*m_db_), Put, writer_options, rocksdb::Slice(key.c_str(), key.size()), rocksdb::Slice(value.buf_, value.buf_len_)); @@ -203,7 +233,9 @@ int RocksDbStoreService::put(void *cf_handle, const std::string &key, const ObSl } else if (is_stopped()) { ret = OB_IN_STOP_STATE; } else { - rocksdb::Status s = m_db_->Put(writer_options, column_family_handle, rocksdb::Slice(key), + rocksdb::Status s; + RETRY_FUNC_ON_IO_ERROR_WITH_USLEEP_MS(is_stopped(), 1 * _SEC_, (*m_db_), Put, writer_options, column_family_handle, + rocksdb::Slice(key), rocksdb::Slice(value.buf_, value.buf_len_)); if (!s.ok()) { @@ -231,10 +263,11 @@ int RocksDbStoreService::batch_write(void *cf_handle, rocksdb::WriteBatch batch; for (int64_t idx = 0; OB_SUCC(ret) && !is_stopped() && idx < keys.size(); ++idx) { - rocksdb::Status s = batch.Put( - column_family_handle, + rocksdb::Status s; + RETRY_FUNC_ON_IO_ERROR_WITH_USLEEP_MS(is_stopped(), 1 * _SEC_, batch, Put, column_family_handle, rocksdb::Slice(keys[idx]), rocksdb::Slice(values[idx].buf_, values[idx].buf_len_)); + if (!s.ok()) { ret = OB_IO_ERROR; _LOG_ERROR("RocksDbStoreService build batch failed, error %s", s.ToString().c_str()); @@ -313,7 +346,9 @@ int RocksDbStoreService::del(const std::string &key) // find column family handle for cf rocksdb::WriteOptions writer_options; writer_options.disableWAL = true; - rocksdb::Status s = m_db_->Delete(writer_options, key); + rocksdb::Status s; + RETRY_FUNC_ON_IO_ERROR_WITH_USLEEP_MS(is_stopped(), 1 * _SEC_, (*m_db_), Delete, writer_options, key); + if (!s.ok()) { ret = OB_IO_ERROR; _LOG_ERROR("delete %s from rocksdb failed, error %s", key.c_str(), s.ToString().c_str()); @@ -336,7 +371,9 @@ int RocksDbStoreService::del(void *cf_handle, const std::string &key) } else if (is_stopped()) { ret = OB_IN_STOP_STATE; } else { - rocksdb::Status s = m_db_->Delete(writer_options, column_family_handle, key); + rocksdb::Status s; + RETRY_FUNC_ON_IO_ERROR_WITH_USLEEP_MS(is_stopped(), 1 * _SEC_, (*m_db_), Delete, writer_options, + column_family_handle, key); if (!s.ok()) { LOG_ERROR("delete %s from rocksdb failed, error %s", key.c_str(), s.ToString().c_str()); @@ -361,8 +398,9 @@ int RocksDbStoreService::del_range(void *cf_handle, const std::string &begin_key } else { rocksdb::WriteOptions writer_options; writer_options.disableWAL = true; - rocksdb::Status s = m_db_->DeleteRange(writer_options, column_family_handle, - begin_key, end_key); + rocksdb::Status s; + RETRY_FUNC_ON_IO_ERROR_WITH_USLEEP_MS(is_stopped(), 1 * _SEC_, (*m_db_), DeleteRange, writer_options, + column_family_handle, begin_key, end_key); if (!s.ok()) { LOG_ERROR("DeleteRange %s from rocksdb failed, error %s", begin_key.c_str(), s.ToString().c_str()); diff --git a/src/logservice/logfetcher/ob_log_fetch_log_rpc.cpp b/src/logservice/logfetcher/ob_log_fetch_log_rpc.cpp index 395acce96d..d2799cd72c 100644 --- a/src/logservice/logfetcher/ob_log_fetch_log_rpc.cpp +++ b/src/logservice/logfetcher/ob_log_fetch_log_rpc.cpp @@ -1133,34 +1133,22 @@ int FetchLogARpc::analyze_result_(RpcRequest &rpc_req, ERRSIM_POINT_DEF(ALLOC_FETCH_LOG_ARPC_CB_FAIL); rpc::frame::ObReqTransport::AsyncCB *FetchLogARpc::RpcCB::clone(const rpc::frame::SPAlloc &alloc) const { - void *buf = NULL; - RpcCB *cb = NULL; - if (OB_SUCCESS != ALLOC_FETCH_LOG_ARPC_CB_FAIL) { - LOG_ERROR_RET(ALLOC_FETCH_LOG_ARPC_CB_FAIL, "ALLOC_FETCH_LOG_ARPC_CB_FAIL"); - } else if (OB_ISNULL(buf = alloc(sizeof(RpcCB)))) { - LOG_ERROR_RET(OB_ALLOCATE_MEMORY_FAILED, "clone rpc callback fail", K(buf), K(sizeof(RpcCB))); - } else if (OB_ISNULL(cb = new(buf) RpcCB(host_))) { - LOG_ERROR_RET(OB_ALLOCATE_MEMORY_FAILED, "construct RpcCB fail", K(buf)); - } else { - // 成功 - } - - return cb; + return static_cast(const_cast(this)); } int FetchLogARpc::RpcCB::process() { int ret = OB_SUCCESS; ObCdcLSFetchLogResp &result = RpcCBBase::result_; - ObRpcResultCode &rcode = RpcCBBase::rcode_; - const common::ObAddr &svr = RpcCBBase::dst_; + const ObRpcResultCode rcode = RpcCBBase::rcode_; + const common::ObAddr svr = RpcCBBase::dst_; if (OB_FAIL(do_process_(rcode, &result))) { - LOG_ERROR("process fetch log callback fail", KR(ret), K(result), K(rcode), K(svr)); + LOG_ERROR("process fetch log callback fail", KR(ret), K(rcode), K(svr)); } // Aone: // Note: Active destructe response after asynchronous RPC processing - result.reset(); + // result.reset(); return ret; } @@ -1169,7 +1157,7 @@ void FetchLogARpc::RpcCB::on_timeout() { int ret = OB_SUCCESS; ObRpcResultCode rcode; - const common::ObAddr &svr = RpcCBBase::dst_; + const common::ObAddr svr = RpcCBBase::dst_; rcode.rcode_ = OB_TIMEOUT; (void)snprintf(rcode.msg_, sizeof(rcode.msg_), "fetch log rpc timeout, svr=%s", @@ -1184,7 +1172,7 @@ void FetchLogARpc::RpcCB::on_invalid() { int ret = OB_SUCCESS; ObRpcResultCode rcode; - const common::ObAddr &svr = RpcCBBase::dst_; + const common::ObAddr svr = RpcCBBase::dst_; // Invalid package encountered, decode failed rcode.rcode_ = OB_RPC_PACKET_INVALID; @@ -1204,7 +1192,7 @@ int FetchLogARpc::RpcCB::do_process_(const ObRpcResultCode &rcode, const ObCdcLS FetchLogARpc &rpc_host = rpc_req.host_; if (OB_FAIL(rpc_host.handle_rpc_response(rpc_req, rcode, resp))) { - LOG_ERROR("set fetch log response fail", KR(ret), K(resp), K(rcode)); + LOG_ERROR("set fetch log response fail", KR(ret), K(rcode)); } else { // success } diff --git a/src/logservice/logfetcher/ob_log_fetch_log_rpc.h b/src/logservice/logfetcher/ob_log_fetch_log_rpc.h index efeb910448..0e95e4399e 100644 --- a/src/logservice/logfetcher/ob_log_fetch_log_rpc.h +++ b/src/logservice/logfetcher/ob_log_fetch_log_rpc.h @@ -297,7 +297,7 @@ private: { public: explicit RpcCB(RpcRequest &host) : host_(host) {} - virtual ~RpcCB() {} + virtual ~RpcCB() { result_.reset(); } public: rpc::frame::ObReqTransport::AsyncCB *clone(const rpc::frame::SPAlloc &alloc) const; diff --git a/src/logservice/palf/log_config_mgr.h b/src/logservice/palf/log_config_mgr.h index d9559dd3af..8ecede47b3 100755 --- a/src/logservice/palf/log_config_mgr.h +++ b/src/logservice/palf/log_config_mgr.h @@ -189,7 +189,7 @@ inline bool is_paxos_member_list_change(const LogConfigChangeType type) return (ADD_MEMBER == type || REMOVE_MEMBER == type || ADD_MEMBER_AND_NUM == type || REMOVE_MEMBER_AND_NUM == type || SWITCH_LEARNER_TO_ACCEPTOR == type || SWITCH_ACCEPTOR_TO_LEARNER == type - || CHANGE_REPLICA_NUM == type); + || CHANGE_REPLICA_NUM == type || SWITCH_LEARNER_TO_ACCEPTOR_AND_NUM == type); } inline bool is_try_lock_config_change(const LogConfigChangeType type) diff --git a/src/logservice/palf/log_sliding_window.cpp b/src/logservice/palf/log_sliding_window.cpp index 298e483c7c..d4bbadc905 100644 --- a/src/logservice/palf/log_sliding_window.cpp +++ b/src/logservice/palf/log_sliding_window.cpp @@ -3507,6 +3507,7 @@ int LogSlidingWindow::submit_group_log(const LSN &lsn, // get log_task success } if (OB_SUCC(ret)) { + log_task->lock(); SCN min_scn; if (log_task->is_valid()) { if (lsn != log_task->get_begin_lsn() @@ -3530,7 +3531,6 @@ int LogSlidingWindow::submit_group_log(const LSN &lsn, PALF_LOG(WARN, "try_update_max_lsn_ failed", K(ret), K_(palf_id), K_(self), K(lsn), K(group_entry_header)); } else { // prev_log_proposal_id match or not exist, receive this log - log_task->lock(); if (log_task->is_valid()) { // log_task可能被其他线程并发收取了,预期内容与本线程一致. if (group_entry_header.get_log_proposal_id() != log_task->get_proposal_id()) { @@ -3547,11 +3547,11 @@ int LogSlidingWindow::submit_group_log(const LSN &lsn, (void) log_task->set_freezed(); log_task->set_freeze_ts(ObTimeUtility::current_time()); } - log_task->unlock(); PALF_LOG(TRACE, "submit_group_log", K(ret), K_(palf_id), K_(self), K(group_entry_header), K(log_id), KPC(log_task)); } + log_task->unlock(); } } } diff --git a/src/logservice/restoreservice/ob_log_restore_driver_base.h b/src/logservice/restoreservice/ob_log_restore_driver_base.h index fece8a97f3..941e055217 100644 --- a/src/logservice/restoreservice/ob_log_restore_driver_base.h +++ b/src/logservice/restoreservice/ob_log_restore_driver_base.h @@ -29,7 +29,7 @@ namespace logservice class ObLogService; class ObLogRestoreDriverBase { - const int64_t FETCH_LOG_AHEAD_THRESHOLD_NS = 3 * 1000 * 1000 *1000L; // 3s + const int64_t FETCH_LOG_AHEAD_THRESHOLD_NS = 6 * 1000 * 1000 *1000L; // 6s public: ObLogRestoreDriverBase(); virtual ~ObLogRestoreDriverBase(); diff --git a/src/objit/include/objit/common/ob_item_type.h b/src/objit/include/objit/common/ob_item_type.h index 3f7fc1d091..bf55f620cb 100755 --- a/src/objit/include/objit/common/ob_item_type.h +++ b/src/objit/include/objit/common/ob_item_type.h @@ -81,6 +81,7 @@ typedef enum ObItemType T_JSON = 47, T_GEOMETRY = 48, T_UDT_SQL = 49, + T_COLLECTION = 51, T_ROARINGBITMAP = 52, T_IEEE754_NAN = 61, @@ -853,6 +854,7 @@ typedef enum ObItemType T_FUN_SYS_XML_EXISTSNODE = 1734, T_FUN_SYS_PRIV_ST_GEOHASH = 1735, T_FUN_SYS_PRIV_ST_MAKEPOINT = 1736, + T_FUN_SYS_ARRAY = 1737, ///< @note add new oracle only function type before this line T_FUN_SYS_TABLET_AUTOINC_NEXTVAL = 1801, // add only for heap table @@ -911,6 +913,9 @@ typedef enum ObItemType T_FUN_SYS_RB_AND_NULL2EMPTY = 2043, T_FUN_SYS_RB_OR_NULL2EMPTY = 2044, T_FUN_SYS_RB_ANDNOT_NULL2EMPTY = 2045, + T_FUN_SYS_RB_TO_STRING = 2046, + T_FUN_SYS_RB_FROM_STRING = 2047, + T_FUN_SYS_RB_ITERATE = 2048, T_MAX_OP = 3000, //pseudo column, to mark the group iterator id diff --git a/src/observer/mysql/obmp_stmt_prexecute.cpp b/src/observer/mysql/obmp_stmt_prexecute.cpp index db9e766b99..8364df2aff 100644 --- a/src/observer/mysql/obmp_stmt_prexecute.cpp +++ b/src/observer/mysql/obmp_stmt_prexecute.cpp @@ -62,6 +62,13 @@ using namespace sql; using namespace pl; namespace observer { + +#ifdef ERRSIM +ERRSIM_POINT_DEF(COM_STMT_PREXECUTE_PREPARE_ERROR); +ERRSIM_POINT_DEF(COM_STMT_PREXECUTE_PS_CURSOR_OPEN_ERROR); +ERRSIM_POINT_DEF(COM_STMT_PREXECUTE_EXECUTE_ERROR); +#endif + ObMPStmtPrexecute::ObMPStmtPrexecute(const ObGlobalContext &gctx) : ObMPStmtExecute(gctx), sql_(), @@ -244,10 +251,15 @@ int ObMPStmtPrexecute::before_process() LOG_WARN("fail to set session active", K(ret)); } if (OB_SUCC(ret)) { - if (OB_FAIL(gctx_.sql_engine_->stmt_prepare(sql_, + if ( +#ifdef ERRSIM + OB_FAIL(COM_STMT_PREXECUTE_PREPARE_ERROR) || +#endif + OB_FAIL(gctx_.sql_engine_->stmt_prepare(sql_, get_ctx(), result, - false/*is_inner_sql*/))) { + false /*is_inner_sql*/)) + ) { set_exec_start_timestamp(ObTimeUtility::current_time()); int cli_ret = OB_SUCCESS; get_retry_ctrl().test_and_save_retry_state(gctx_, @@ -473,7 +485,12 @@ int ObMPStmtPrexecute::execute_response(ObSQLSessionInfo &session, ObPLExecCtx pl_ctx(cursor->get_allocator(), &result.get_exec_context(), NULL/*params*/, NULL/*result*/, &ret, NULL/*func*/, true); get_ctx().cur_sql_ = sql_; - if (OB_FAIL(ObSPIService::dbms_dynamic_open(&pl_ctx, *cursor))) { + if ( +#ifdef ERRSIM + OB_FAIL(COM_STMT_PREXECUTE_PS_CURSOR_OPEN_ERROR) || +#endif + OB_FAIL(ObSPIService::dbms_dynamic_open(&pl_ctx, *cursor)) + ) { LOG_WARN("cursor open faild.", K(cursor->get_id())); // select do not support arraybinding if (!THIS_WORKER.need_retry()) { @@ -597,12 +614,17 @@ int ObMPStmtPrexecute::execute_response(ObSQLSessionInfo &session, ret = tmp_ret; LOG_WARN("execute server cursor failed.", K(ret)); } - } else if (OB_FAIL(gctx_.sql_engine_->stmt_execute(stmt_id_, - stmt_type_, - params, - ctx, - result, - false /* is_inner_sql */))) { + } else if ( +#ifdef ERRSIM + OB_FAIL(COM_STMT_PREXECUTE_EXECUTE_ERROR) || +#endif + OB_FAIL(gctx_.sql_engine_->stmt_execute(stmt_id_, + stmt_type_, + params, + ctx, + result, + false /* is_inner_sql */)) + ) { set_exec_start_timestamp(ObTimeUtility::current_time()); if (!THIS_WORKER.need_retry()) { int cli_ret = OB_SUCCESS; diff --git a/src/observer/mysql/obsm_utils.cpp b/src/observer/mysql/obsm_utils.cpp index 5bc7069e28..2b8c4ab7a1 100644 --- a/src/observer/mysql/obsm_utils.cpp +++ b/src/observer/mysql/obsm_utils.cpp @@ -93,6 +93,7 @@ static const ObMySQLTypeMap type_maps_[ObMaxType] = {EMySQLFieldType::MYSQL_TYPE_GEOMETRY, BLOB_FLAG | BINARY_FLAG, 0}, /* ObGeometryType */ {EMySQLFieldType::MYSQL_TYPE_COMPLEX, 0, 0}, /* ObUserDefinedSQLType */ {EMySQLFieldType::MYSQL_TYPE_NEWDECIMAL, 0, 0}, /* ObDecimalIntType */ + {EMySQLFieldType::MYSQL_TYPE_STRING, 0, 0}, /* ObCollectionSQLType, will cast to string */ /* ObMaxType */ }; diff --git a/src/observer/table/ob_table_query_async_processor.cpp b/src/observer/table/ob_table_query_async_processor.cpp index 7ddbd7d473..e279173f69 100644 --- a/src/observer/table/ob_table_query_async_processor.cpp +++ b/src/observer/table/ob_table_query_async_processor.cpp @@ -377,7 +377,7 @@ int ObTableQueryAsyncP::get_query_session(uint64_t sessid, ObTableQueryAsyncSess LOG_WARN("fail to insert session to query map", K(ret), K(sessid)); OB_DELETE(ObTableQueryAsyncSession, ObModIds::TABLE_PROC, query_session); } else {} - } else if (ObQueryOperationType::QUERY_NEXT == arg_.query_type_) { + } else if (ObQueryOperationType::QUERY_NEXT == arg_.query_type_ || ObQueryOperationType::QUERY_END == arg_.query_type_) { if (OB_FAIL(ObQueryAsyncMgr::get_instance().get_query_session(sessid, query_session))) { LOG_WARN("fail to get query session from query sync mgr", K(ret), K(sessid)); } else if (OB_ISNULL(query_session)) { @@ -609,6 +609,13 @@ int ObTableQueryAsyncP::process_query_next() return ret; } +int ObTableQueryAsyncP::process_query_end() +{ + int ret = OB_SUCCESS; + result_.is_end_ = true; + return ret; +} + int ObTableQueryAsyncP::try_process() { int ret = OB_SUCCESS; @@ -627,6 +634,8 @@ int ObTableQueryAsyncP::try_process() ret = process_query_start(); } else if (ObQueryOperationType::QUERY_NEXT == arg_.query_type_) { ret = process_query_next(); + } else if (ObQueryOperationType::QUERY_END == arg_.query_type_) { + ret = process_query_end(); } if (OB_FAIL(ret)) { LOG_WARN("query execution failed, need rollback", K(ret)); @@ -689,8 +698,8 @@ int ObTableQueryAsyncP::destory_query_session(bool need_rollback_trans) int ObTableQueryAsyncP::check_query_type() { int ret = OB_SUCCESS; - if (arg_.query_type_ != table::ObQueryOperationType::QUERY_START && - arg_.query_type_ != table::ObQueryOperationType::QUERY_NEXT){ + if (arg_.query_type_ < table::ObQueryOperationType::QUERY_START || + arg_.query_type_ >= table::ObQueryOperationType::QUERY_MAX) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid query operation type", K(ret), K(arg_.query_type_)); } diff --git a/src/observer/table/ob_table_query_async_processor.h b/src/observer/table/ob_table_query_async_processor.h index b416cce31c..32bd4031b2 100644 --- a/src/observer/table/ob_table_query_async_processor.h +++ b/src/observer/table/ob_table_query_async_processor.h @@ -226,6 +226,7 @@ protected: private: int process_query_start(); int process_query_next(); + int process_query_end(); int destory_query_session(bool need_rollback_trans); DISALLOW_COPY_AND_ASSIGN(ObTableQueryAsyncP); diff --git a/src/observer/table_load/ob_table_load_store.cpp b/src/observer/table_load/ob_table_load_store.cpp index 46c23937d3..53088f0dbe 100644 --- a/src/observer/table_load/ob_table_load_store.cpp +++ b/src/observer/table_load/ob_table_load_store.cpp @@ -230,8 +230,11 @@ public: } else if (OB_FAIL(ctx_->store_ctx_->check_status(ObTableLoadStatusType::INITED))) { LOG_WARN("fail to check status", KR(ret)); } else if (OB_FAIL(tablet_ctx->open())) { - LOG_WARN("fail to open tablet context", KR(ret)); - ret = OB_SUCCESS; + LOG_WARN("fail to open tablet context", KR(ret), K(tablet_id)); + if (ret == OB_EAGAIN) { + LOG_WARN("retry to open tablet context", K(tablet_id)); + ret = OB_SUCCESS; + } } else { ctx_->store_ctx_->handle_open_insert_tablet_ctx_finish(is_finish); break; diff --git a/src/observer/virtual_table/ob_all_virtual_tx_data.cpp b/src/observer/virtual_table/ob_all_virtual_tx_data.cpp index 27e7650d13..bf8e298452 100644 --- a/src/observer/virtual_table/ob_all_virtual_tx_data.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tx_data.cpp @@ -101,6 +101,10 @@ int ObAllVirtualTxData::fill_in_row_(const VirtualTxDataRow &row_data, common::O cur_row_.cells_[i].set_varchar(row_data.undo_status_list_str_); cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); break; + case TX_OP_COL: + cur_row_.cells_[i].set_varchar(row_data.tx_op_str_); + cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + break; default: ret = OB_ERR_UNEXPECTED; break; @@ -207,4 +211,4 @@ int ObAllVirtualTxData::generate_virtual_tx_data_row_(VirtualTxDataRow &tx_data_ } } // namespace observer -} // namespace oceanbase \ No newline at end of file +} // namespace oceanbase diff --git a/src/observer/virtual_table/ob_all_virtual_tx_data.h b/src/observer/virtual_table/ob_all_virtual_tx_data.h index f02bb67c68..15566a58ed 100644 --- a/src/observer/virtual_table/ob_all_virtual_tx_data.h +++ b/src/observer/virtual_table/ob_all_virtual_tx_data.h @@ -35,10 +35,11 @@ struct VirtualTxDataRow { share::SCN end_scn_; share::SCN commit_version_; char undo_status_list_str_[common::MAX_UNDO_LIST_CHAR_LENGTH]; + char tx_op_str_[common::MAX_TX_OP_CHAR_LENGTH]; VirtualTxDataRow() : state_(0), start_scn_(), end_scn_(), commit_version_() {} - TO_STRING_KV(K(state_), K(start_scn_), K(end_scn_), K(commit_version_), K(undo_status_list_str_)); + TO_STRING_KV(K(state_), K(start_scn_), K(end_scn_), K(commit_version_), K(undo_status_list_str_), K(tx_op_str_)); }; class ObAllVirtualTxData : public common::ObVirtualTableScannerIterator { @@ -53,7 +54,8 @@ private: START_SCN_COL, END_SCN_COL, COMMIT_VERSION_COL, - UNDO_STATUS_COL + UNDO_STATUS_COL, + TX_OP_COL }; diff --git a/src/pl/ob_pl.cpp b/src/pl/ob_pl.cpp index df09504bb6..8bc4f88942 100644 --- a/src/pl/ob_pl.cpp +++ b/src/pl/ob_pl.cpp @@ -2873,6 +2873,17 @@ int ObPLExecCtx::get_user_type(uint64_t type_id, return ret; } +int ObPLExecCtx::calc_expr(uint64_t package_id, int64_t expr_idx, ObObjParam &result) +{ + int ret = OB_SUCCESS; + if (OB_INVALID_ID == package_id) { + OZ (ObSPIService::spi_calc_expr_at_idx(this, expr_idx, OB_INVALID_INDEX, &result)); + } else { + OZ (ObSPIService::spi_calc_package_expr(this, package_id, expr_idx, &result)); + } + return ret; +} + int ObPLExecState::final(int ret) { int tmp_ret = OB_SUCCESS; @@ -3971,7 +3982,8 @@ int ObPLExecState::check_pl_execute_priv(ObSchemaGetterGuard &guard, OBJ_PRIV_ID_EXECUTE, CHECK_FLAG_NORMAL, obj_owner_id, - role_id_array)); + role_id_array), + K(obj_tenant_id), K(user_id), K(database_name), K(obj_id), K(obj_owner_id), K(role_id_array)); } if (ROUTINE_SCHEMA == schema_type && ret == OB_TABLE_NOT_EXIST) { ret = OB_WRONG_COLUMN_NAME; @@ -4453,11 +4465,18 @@ int ObPLFunction::is_special_pkg_invoke_right(ObSchemaGetterGuard &guard, bool & return ret; } +int ObPLINS::calc_expr(uint64_t package_id, int64_t expr_idx, ObObjParam &result) +{ + int ret = OB_NOT_SUPPORTED; + LOG_USER_WARN(OB_NOT_SUPPORTED, "call expr on base class ObIPLNS"); + return ret; +} + int ObPLINS::init_complex_obj(ObIAllocator &allocator, const ObPLDataType &pl_type, common::ObObjParam &obj, bool set_allocator, - bool set_null) const + bool set_null) { int ret = OB_SUCCESS; int64_t init_size = 0; @@ -4489,16 +4508,36 @@ int ObPLINS::init_complex_obj(ObIAllocator &allocator, OX (record = reinterpret_cast(ptr)); for (int64_t i = 0; OB_SUCC(ret) && i < record_type->get_member_count(); ++i) { CK (OB_NOT_NULL(record_type->get_member(i))); + CK (OB_NOT_NULL(record_type->get_record_member(i))); OZ (record->get_element(i, member)); CK (OB_NOT_NULL(member)); - if (record_type->get_member(i)->is_obj_type()) { - OX (new (member) ObObj(ObNullType)); + OX (new (member) ObObj(ObNullType)); + if (OB_FAIL(ret)) { + } else if (record_type->get_record_member(i)->get_default() != OB_INVALID_INDEX) { + ObObjParam default_v; + if (record_type->is_package_type()) { + OZ (calc_expr(extract_package_id(pl_type.get_user_type_id()), + record_type->get_record_member(i)->get_default(), + default_v)); + } else { + OZ (calc_expr(OB_INVALID_ID, + record_type->get_record_member(i)->get_default(), + default_v)); + } + if (OB_FAIL(ret)) { + } else if (record_type->get_member(i)->is_obj_type()) { + OZ (deep_copy_obj(allocator, default_v, *member)); + } else { + OZ (ObUserDefinedType::deep_copy_obj(allocator, default_v, *member)); + } } else { - int64_t init_size = OB_INVALID_SIZE; - int64_t member_ptr = 0; - OZ (record_type->get_member(i)->get_size(PL_TYPE_INIT_SIZE, init_size)); - OZ (record_type->get_member(i)->newx(allocator, this, member_ptr)); - OX (member->set_extend(member_ptr, record_type->get_member(i)->get_type(), init_size)); + if (!record_type->get_member(i)->is_obj_type()) { + int64_t init_size = OB_INVALID_SIZE; + int64_t member_ptr = 0; + OZ (record_type->get_member(i)->get_size(PL_TYPE_INIT_SIZE, init_size)); + OZ (record_type->get_member(i)->newx(allocator, this, member_ptr)); + OX (member->set_extend(member_ptr, record_type->get_member(i)->get_type(), init_size)); + } } } // f(self object_type, p1 out object_type), p1 will be init here, we have to set it null diff --git a/src/pl/ob_pl.h b/src/pl/ob_pl.h index 6f2cbaa71b..316b88f018 100644 --- a/src/pl/ob_pl.h +++ b/src/pl/ob_pl.h @@ -95,7 +95,9 @@ public: const ObPLDataType &pl_type, common::ObObjParam &obj, bool set_allocator = false, - bool set_null = true) const; + bool set_null = true); + + virtual int calc_expr(uint64_t package_id, int64_t expr_idx, ObObjParam &result); }; class ObPLFunctionBase @@ -645,8 +647,8 @@ struct ObPLExecCtx : public ObPLINS virtual int get_user_type(uint64_t type_id, const ObUserDefinedType *&user_type, ObIAllocator *allocator = NULL) const; + virtual int calc_expr(uint64_t package_id, int64_t expr_idx, ObObjParam &result); - //Note: 不实现虚函数，省得llvm解析的时候需要处理vtable麻烦 common::ObIAllocator *allocator_; sql::ObExecContext *exec_ctx_; ParamStore *params_; // param stroe, 对应PL Function的符号表 diff --git a/src/pl/ob_pl_resolver.cpp b/src/pl/ob_pl_resolver.cpp index 9f05e13dbf..0c1d57bc1a 100644 --- a/src/pl/ob_pl_resolver.cpp +++ b/src/pl/ob_pl_resolver.cpp @@ -2635,9 +2635,11 @@ int ObPLResolver::collect_dep_info_by_schema(const ObPLResolveCtx &ctx, if (table_schema->is_view_table() && !table_schema->is_materialized_view()) { OZ (collect_dep_info_by_view_schema(ctx, table_schema, dependency_objects)); } else { - OZ(dependency_objects.push_back(ObSchemaObjVersion(table_schema->get_table_id(), - table_schema->get_schema_version(), - ObDependencyTableType::DEPENDENCY_TABLE))); + ObSchemaObjVersion version(table_schema->get_table_id(), + table_schema->get_schema_version(), + ObDependencyTableType::DEPENDENCY_TABLE); + version.is_db_explicit_ = ctx.session_info_.get_database_id() != table_schema->get_database_id(); + OZ(dependency_objects.push_back(version)); } } return ret; @@ -2674,9 +2676,11 @@ int ObPLResolver::build_record_type_by_schema( OZ (build_record_type_by_table_schema( resolve_ctx.schema_guard_, resolve_ctx.allocator_, table_schema, record_type, with_rowid)); if (OB_NOT_NULL(dependency_objects)) { - OZ(dependency_objects->push_back(ObSchemaObjVersion(table_schema->get_table_id(), - table_schema->get_schema_version(), - ObDependencyTableType::DEPENDENCY_TABLE))); + ObSchemaObjVersion version(table_schema->get_table_id(), + table_schema->get_schema_version(), + ObDependencyTableType::DEPENDENCY_TABLE); + version.is_db_explicit_ = resolve_ctx.session_info_.get_database_id() != table_schema->get_database_id(); + OZ(dependency_objects->push_back(version)); } } } diff --git a/src/pl/ob_pl_stmt.cpp b/src/pl/ob_pl_stmt.cpp index 6b1acdd5eb..511d106b5d 100644 --- a/src/pl/ob_pl_stmt.cpp +++ b/src/pl/ob_pl_stmt.cpp @@ -2217,9 +2217,9 @@ int ObPLExternalNS::resolve_external_routine(const ObString &db_name, LOG_WARN("add dependency object failed", "package_id", schema_routine_info->get_package_id(), K(ret)); } else if (synonym_checker.has_synonym()) { if (OB_FAIL(ObResolverUtils::add_dependency_synonym_object(&resolve_ctx_.schema_guard_, - &resolve_ctx_.session_info_, - synonym_checker, - *get_dependency_table()))) { + &resolve_ctx_.session_info_, + synonym_checker, + *get_dependency_table()))) { LOG_WARN("add dependency synonym failed", K(ret)); } } diff --git a/src/pl/ob_pl_stmt.h b/src/pl/ob_pl_stmt.h index 5bc45fe3a0..b32327408f 100644 --- a/src/pl/ob_pl_stmt.h +++ b/src/pl/ob_pl_stmt.h @@ -1208,6 +1208,8 @@ public: inline const ObPLBlockNS *get_parent_ns() const { return parent_ns_; } inline const ObPLResolveCtx &get_resolve_ctx() { return resolve_ctx_; } inline const ObPLDependencyTable *get_dependency_table() const { return dependency_table_; } + + inline ObPLDependencyTable *get_dependency_table() { return dependency_table_; } inline void set_dependency_table(ObPLDependencyTable *dependency_table) { dependency_table_ = dependency_table; } int add_dependency_object(const share::schema::ObSchemaObjVersion &obj_version) const; diff --git a/src/pl/pl_cache/ob_pl_cache.cpp b/src/pl/pl_cache/ob_pl_cache.cpp index 414232c288..bca2cfc959 100644 --- a/src/pl/pl_cache/ob_pl_cache.cpp +++ b/src/pl/pl_cache/ob_pl_cache.cpp @@ -444,9 +444,11 @@ int ObPLObjectValue::check_value_version(share::schema::ObSchemaGetterGuard *sch OZ (obtain_new_column_infos(*schema_guard, schema_obj2, column_infos)); OX (is_old_version = !schema_obj1->match_columns(column_infos)); } else { - LOG_WARN("mismatched schema objs", K(*schema_obj1), K(schema_obj2), K(i)); is_old_version = true; } + if (OB_SUCC(ret) && is_old_version) { + LOG_WARN("mismatched schema objs", K(*schema_obj1), K(schema_obj2), K(i)); + } } } } @@ -719,6 +721,7 @@ int ObPLObjectValue::match_dep_schema(const ObPLCacheCtx &pc_ctx, && !stored_schema_objs_.at(i)->match_compare(schema_array.at(i))) { // check whether common table name is same as system table in oracle mode is_same = false; + LOG_WARN("mismatched schema objs", K(*stored_schema_objs_.at(i)), K(stored_schema_objs_.at(i)), K(i)); } else { // do nothing } diff --git a/src/pl/sys_package/ob_dbms_stats.cpp b/src/pl/sys_package/ob_dbms_stats.cpp index 9cb267dbf8..ccefafe036 100644 --- a/src/pl/sys_package/ob_dbms_stats.cpp +++ b/src/pl/sys_package/ob_dbms_stats.cpp @@ -32,6 +32,7 @@ #include "share/stat/ob_opt_stat_gather_stat.h" #include "sql/engine/expr/ob_expr_uuid.h" #include "sql/privilege_check/ob_ora_priv_check.h" +#include "sql/ob_result_set.h" namespace oceanbase { @@ -76,6 +77,8 @@ int ObDbmsStats::gather_table_stats(ObExecContext &ctx, ParamStore ¶ms, ObOb int64_t start_time = ObTimeUtility::current_time(); if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_ISNULL(ctx.get_my_session()) || OB_ISNULL(ctx.get_task_executor_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(ctx.get_my_session()), K(ctx.get_task_executor_ctx())); @@ -176,6 +179,8 @@ int ObDbmsStats::gather_schema_stats(ObExecContext &ctx, ParamStore ¶ms, ObO int64_t start_time = ObTimeUtility::current_time(); if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_ISNULL(ctx.get_my_session()) || OB_ISNULL(ctx.get_task_executor_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(ctx.get_my_session()), K(ctx.get_task_executor_ctx())); @@ -309,6 +314,8 @@ int ObDbmsStats::gather_index_stats(ObExecContext &ctx, ParamStore ¶ms, ObOb empty_cascade.set_null(); if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (lib::is_oracle_mode() && !params.at(11).is_null()) { ret = OB_ERR_DBMS_STATS_PL; LOG_WARN("table name shouldn't be specified in gather index stats", K(ret)); @@ -528,6 +535,8 @@ int ObDbmsStats::set_table_stats(ObExecContext &ctx, ParamStore ¶ms, ObObj & param.table_param_.allocator_ = &ctx.get_allocator(); if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_FAIL(parse_set_table_info(ctx, params.at(0), params.at(1), @@ -606,6 +615,8 @@ int ObDbmsStats::set_column_stats(sql::ObExecContext &ctx, param.table_param_.allocator_ = &ctx.get_allocator(); if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (params.at(2).is_null() && !params.at(1).is_null()) { //do nothing } else if (OB_FAIL(parse_set_column_stats(ctx, @@ -613,6 +624,7 @@ int ObDbmsStats::set_column_stats(sql::ObExecContext &ctx, params.at(1), params.at(2), params.at(3), + param.col_meta_, param.table_param_))) { LOG_WARN("failed to parse set column stats", K(ret)); } else if (OB_FAIL(parse_set_column_stats_options(ctx, @@ -700,6 +712,8 @@ int ObDbmsStats::set_index_stats(ObExecContext &ctx, ParamStore ¶ms, ObObj & number::ObNumber num_nummicroblks; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (lib::is_oracle_mode() && !params.at(22).is_null()) { ret = OB_ERR_DBMS_STATS_PL; LOG_WARN("table name shouldn't be specified in gather index stats", K(ret)); @@ -786,6 +800,8 @@ int ObDbmsStats::delete_table_stats(ObExecContext &ctx, ParamStore ¶ms, ObOb bool cascade_indexes = false; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_FAIL(parse_table_part_info(ctx, params.at(0), params.at(1), @@ -866,6 +882,8 @@ int ObDbmsStats::delete_column_stats(ObExecContext &ctx, ParamStore ¶ms, ObO bool only_histogram = false; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_FAIL(parse_table_part_info(ctx, params.at(0), params.at(1), @@ -941,6 +959,8 @@ int ObDbmsStats::delete_schema_stats(ObExecContext &ctx, ParamStore ¶ms, ObO ObSEArray table_ids; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (ctx.get_my_session()->get_is_in_retry()) { ret = OB_ERR_DBMS_STATS_PL; LOG_WARN("retry delete schema stats is not allowed", K(ret)); @@ -1029,6 +1049,8 @@ int ObDbmsStats::delete_index_stats(ObExecContext &ctx, ParamStore ¶ms, ObOb bool only_histogram = false; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (lib::is_oracle_mode() && !params.at(10).is_null()) { ret = OB_ERR_DBMS_STATS_PL; LOG_WARN("table name shouldn't be specified in gather index stats", K(ret)); @@ -1289,6 +1311,8 @@ int ObDbmsStats::export_table_stats(ObExecContext &ctx, ParamStore ¶ms, ObOb const share::schema::ObTableSchema *table_schema = NULL; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_FAIL(parse_table_part_info(ctx, params.at(0), params.at(1), @@ -1367,6 +1391,8 @@ int ObDbmsStats::export_column_stats(sql::ObExecContext &ctx, stat_param.cascade_ = true; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_FAIL(parse_table_part_info(ctx, params.at(0), params.at(1), @@ -1427,6 +1453,8 @@ int ObDbmsStats::export_schema_stats(ObExecContext &ctx, ParamStore ¶ms, ObO ObString tmp_str; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (ctx.get_my_session()->get_is_in_retry()) { ret = OB_ERR_DBMS_STATS_PL; LOG_WARN("retry export schema stats is not allowed", K(ret)); @@ -1506,6 +1534,8 @@ int ObDbmsStats::export_index_stats(ObExecContext &ctx, ParamStore ¶ms, ObOb const share::schema::ObTableSchema *table_schema = NULL; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (lib::is_oracle_mode() && !params.at(6).is_null()) { ret = OB_ERR_DBMS_STATS_PL; LOG_WARN("table name shouldn't be specified in gather index stats", K(ret)); @@ -1615,6 +1645,8 @@ int ObDbmsStats::import_table_stats(ObExecContext &ctx, ParamStore ¶ms, ObOb const share::schema::ObTableSchema *table_schema = NULL; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_FAIL(parse_table_part_info(ctx, params.at(0), params.at(1), @@ -1712,6 +1744,8 @@ int ObDbmsStats::import_column_stats(sql::ObExecContext &ctx, stat_param.cascade_ = true; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_FAIL(parse_table_part_info(ctx, params.at(0), params.at(1), @@ -1783,6 +1817,8 @@ int ObDbmsStats::import_schema_stats(ObExecContext &ctx, ParamStore ¶ms, ObO ObSEArray table_ids; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (ctx.get_my_session()->get_is_in_retry()) { ret = OB_ERR_DBMS_STATS_PL; LOG_WARN("retry import schema stats is not allowed", K(ret)); @@ -1885,6 +1921,8 @@ int ObDbmsStats::import_index_stats(ObExecContext &ctx, ParamStore ¶ms, ObOb const share::schema::ObTableSchema *table_schema = NULL; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (lib::is_oracle_mode() && !params.at(8).is_null()) { ret = OB_ERR_DBMS_STATS_PL; LOG_WARN("table name shouldn't be specified in gather index stats", K(ret)); @@ -2009,6 +2047,8 @@ int ObDbmsStats::lock_table_stats(sql::ObExecContext &ctx, ObString stat_type_str; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_FAIL(parse_table_part_info(ctx, params.at(0), params.at(1), @@ -2060,6 +2100,8 @@ int ObDbmsStats::lock_partition_stats(sql::ObExecContext &ctx, stat_param.stattype_ = StatTypeLocked::PARTITION_ALL_TYPE; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (params.at(2).is_null()) { ret = OB_ERR_DBMS_STATS_PL; LOG_WARN("partition not specified", K(ret)); @@ -2108,6 +2150,8 @@ int ObDbmsStats::lock_schema_stats(sql::ObExecContext &ctx, ObSEArray table_ids; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (ctx.get_my_session()->get_is_in_retry()) { ret = OB_ERR_DBMS_STATS_PL; LOG_WARN("retry lock schema stats is not allowed", K(ret)); @@ -2218,6 +2262,8 @@ int ObDbmsStats::unlock_table_stats(sql::ObExecContext &ctx, stat_param.stattype_ = StatTypeLocked::TABLE_ALL_TYPE; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_FAIL(parse_table_part_info(ctx, params.at(0), params.at(1), @@ -2270,6 +2316,8 @@ int ObDbmsStats::unlock_partition_stats(sql::ObExecContext &ctx, stat_param.stattype_ = StatTypeLocked::PARTITION_ALL_TYPE; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (params.at(2).is_null()) { ret = OB_ERR_DBMS_STATS_PL; LOG_WARN("partition not specified", K(ret)); @@ -2394,6 +2442,8 @@ int ObDbmsStats::restore_table_stats(sql::ObExecContext &ctx, int64_t specify_time = 0; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_FAIL(parse_table_part_info(ctx, params.at(0), params.at(1), @@ -2500,6 +2550,8 @@ int ObDbmsStats::restore_schema_stats(sql::ObExecContext &ctx, int64_t specify_time = 0; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (ctx.get_my_session()->get_is_in_retry()) { ret = OB_ERR_DBMS_STATS_PL; LOG_WARN("retry restore schema stats is not allowed", K(ret)); @@ -2585,6 +2637,8 @@ int ObDbmsStats::purge_stats(sql::ObExecContext &ctx, int64_t specify_time = -1; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (lib::is_oracle_mode()) { if (!params.at(0).is_null() && !params.at(0).is_timestamp_tz()) { ret = OB_INVALID_ARGUMENT; @@ -2643,6 +2697,8 @@ int ObDbmsStats::alter_stats_history_retention(sql::ObExecContext &ctx, double retention_tmp = 0.0; // bugfix: if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (!params.at(0).is_null() && OB_FAIL(params.at(0).get_number(num_retention))) { LOG_WARN("failed to get epc", K(ret)); } else if (!params.at(0).is_null() && @@ -2764,6 +2820,8 @@ int ObDbmsStats::reset_global_pref_defaults(sql::ObExecContext &ctx, UNUSED(result); if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_FAIL(ObDbmsStatsPreferences::reset_global_pref_defaults(ctx))) { LOG_WARN("failed to reset global pref defaults"); } else {/*do nothing*/} @@ -2838,6 +2896,8 @@ int ObDbmsStats::set_global_prefs(sql::ObExecContext &ctx, ObStatPrefs *stat_pref = NULL; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (!params.at(0).is_null() && OB_FAIL(params.at(0).get_string(opt_name))) { LOG_WARN("failed to get string", K(ret), K(params.at(0))); } else if (!params.at(0).is_null() && @@ -2892,6 +2952,8 @@ int ObDbmsStats::set_schema_prefs(sql::ObExecContext &ctx, ObStatPrefs *stat_pref = NULL; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (ctx.get_my_session()->get_is_in_retry()) { ret = OB_ERR_DBMS_STATS_PL; LOG_WARN("retry set schema stats is not allowed", K(ret)); @@ -2956,6 +3018,8 @@ int ObDbmsStats::set_table_prefs(sql::ObExecContext &ctx, bool use_size_auto = false; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_FAIL(parse_table_part_info(ctx, params.at(0), params.at(1), dummy_param, param))) { LOG_WARN("failed to get string", K(ret)); } else if (OB_FAIL(table_ids.push_back(param.table_id_))) { @@ -3015,6 +3079,8 @@ int ObDbmsStats::delete_schema_prefs(sql::ObExecContext &ctx, ObStatPrefs *stat_pref = NULL; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (ctx.get_my_session()->get_is_in_retry()) { ret = OB_ERR_DBMS_STATS_PL; LOG_WARN("retry delete schema stats is not allowed", K(ret)); @@ -3067,6 +3133,8 @@ int ObDbmsStats::delete_table_prefs(sql::ObExecContext &ctx, ObStatPrefs *stat_pref = NULL; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_FAIL(parse_table_part_info(ctx, params.at(0), params.at(1), dummy_param, param))) { LOG_WARN("failed to get string", K(ret)); } else if (OB_FAIL(table_ids.push_back(param.table_id_))) { @@ -3583,6 +3651,7 @@ int ObDbmsStats::parse_set_column_stats(ObExecContext &ctx, const ObObjParam &tab_name, const ObObjParam &colname, const ObObjParam &part_name, + ObObjMeta &col_meta, ObTableStatParam ¶m) { int ret = OB_SUCCESS; @@ -3638,6 +3707,7 @@ int ObDbmsStats::parse_set_column_stats(ObExecContext &ctx, } else { col_param.column_id_ = col->get_column_id(); col_param.cs_type_ = col->get_collation_type(); + col_meta = col->get_meta_type(); col_param.gather_flag_ = 0; col_param.bucket_num_ = -1; if (col->is_index_column()) { @@ -4095,7 +4165,12 @@ int ObDbmsStats::get_default_stat_options(ObExecContext &ctx, } } if (OB_SUCC(ret) && stat_options & StatOptionFlags::OPT_BLOCK_SAMPLE) { - param.sample_info_.set_is_block_sample(false); + ObBlockSamplePrefs *tmp_pref = NULL; + if (OB_FAIL(new_stat_prefs(*param.allocator_, ctx.get_my_session(), ObString(), tmp_pref))) { + LOG_WARN("failed to new stat prefs", K(ret)); + } else if (OB_FAIL(stat_prefs.push_back(tmp_pref))) { + LOG_WARN("failed to push back", K(ret)); + } } if (OB_SUCC(ret) && stat_options & StatOptionFlags::OPT_METHOD_OPT) { ObMethodOptPrefs *tmp_pref = NULL; @@ -4793,10 +4868,8 @@ int ObDbmsStats::parse_set_hist_stats_options(ObExecContext &ctx, number::ObNumber num_eavs; if (!epc.is_null() && OB_FAIL(epc.get_number(num_epc))) { LOG_WARN("failed to get epc", K(ret)); - } else if (!minval.is_null() && OB_FAIL(minval.get_raw(hist_param.minval_))) { - LOG_WARN("failed to get minval", K(ret)); - } else if (!maxval.is_null() && OB_FAIL(maxval.get_raw(hist_param.maxval_))) { - LOG_WARN("failed to get maxval", K(ret)); + } else if (!minval.is_null() && FALSE_IT(hist_param.minval_ = &minval)) { + } else if (!maxval.is_null() && FALSE_IT(hist_param.maxval_ = &maxval)) { } else if (OB_FAIL(parser_pl_numarray(bkvals, hist_param.bkvals_))) { LOG_WARN("failed to parser pl numarray", K(ret)); } else if (OB_FAIL(parser_pl_numarray(novals, hist_param.novals_))) { @@ -5369,6 +5442,8 @@ int ObDbmsStats::gather_database_stats_job_proc(sql::ObExecContext &ctx, if (OB_FAIL(check_statistic_table_writeable(ctx))) { ret = OB_SUCCESS; LOG_INFO("auto gather database statistics abort because of statistic table is unwriteable"); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (!ctx.get_my_session()->is_user_session() && no_auto_gather) { //do nothing LOG_INFO("auto gather stat abort because of the trace point and not user seesion", @@ -5424,53 +5499,28 @@ int ObDbmsStats::gather_database_table_stats(sql::ObExecContext &ctx, } else if (OB_FALSE_IT(tenant_id = session->get_effective_tenant_id())) { } else if (is_virtual_tenant_id(tenant_id)) { // do nothing - } else if (OB_FAIL(ObBasicStatsEstimator::get_need_stats_table_cnt(ctx, tenant_id, - task_info.task_table_count_))) { - LOG_WARN("failed to get all tables count", K(ret)); } else { int64_t slice_cnt = 10000; // maximum tables we can gather stats at each iteration - int64_t tmp_succeed = 0; + int64_t offset = 0; do { table_ids.reuse(); - tmp_succeed = succeed_cnt; - if (OB_FAIL(THIS_WORKER.check_status())) { - LOG_WARN("check status failed", KR(ret)); - } else if (OB_FAIL(ObBasicStatsEstimator::get_need_stats_tables(ctx, tenant_id, table_ids, slice_cnt))) { - LOG_WARN("failed to get tables that need gather stats", K(ret)); - } else if (OB_FAIL(do_gather_tables_stats(ctx, tenant_id, table_ids, - duration_time, succeed_cnt, task_info))) { - LOG_WARN("failed to gather table stats", K(ret)); - } - LOG_INFO("succeed to gather table stats", K(ret), K(table_ids.count()), K(slice_cnt), - K(tmp_succeed), K(duration_time), K(succeed_cnt)); - // case that we can break the loop: - // 1. #table_ids < slice_cnt, which means that we have fetched all the tables we need to gather stats - // 2. duration_time_ = -1, and has reached the ob_query_timeout session variable limit - // 3. duration_time is not -1, and the time we cost to gather stats has reached duration_time - } while (OB_SUCC(ret) && table_ids.count() == slice_cnt && (succeed_cnt - tmp_succeed) != 0); - // gather virtual table stats - ObSEArray all_table_ids; - if (OB_FAIL(ret)) { - } else if (OB_ISNULL(ctx.get_virtual_table_ctx().schema_guard_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(ctx.get_virtual_table_ctx().schema_guard_)); - } else if (OB_FAIL(ctx.get_virtual_table_ctx().schema_guard_->get_table_ids_in_tenant(tenant_id, all_table_ids))) { - LOG_WARN("failed to get virtual table ids in tenant", K(ret)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < all_table_ids.count(); ++i) { - int64_t table_id = static_cast(all_table_ids.at(i)); - if (is_virtual_table(table_id) && !ObDbmsStatsUtils::is_no_stat_virtual_table(table_id)) { - if (OB_FAIL(refresh_tenant_schema_guard(ctx, tenant_id))) { + if (OB_FAIL(ObBasicStatsEstimator::get_need_stats_tables(ctx, tenant_id, offset, slice_cnt, table_ids))) { + LOG_WARN("failed to get need stats tables", K(ret)); + } else { + task_info.task_table_count_ += table_ids.count(); + for (int64_t i = 0; OB_SUCC(ret) && i < table_ids.count(); ++i) { + if (OB_FAIL(THIS_WORKER.check_status())) { + LOG_WARN("failed to check status", K(ret)); + } else if (OB_FAIL(refresh_tenant_schema_guard(ctx, tenant_id))) { LOG_WARN("refresh tenant schema guard failed", K(ret)); - } else if (OB_FAIL(do_gather_table_stats(ctx, table_id, tenant_id, + } else if (OB_FAIL(do_gather_table_stats(ctx, table_ids.at(i), tenant_id, duration_time, succeed_cnt, task_info))) { - LOG_WARN("failed to gather virtual table stats", K(ret)); - } else { - ++task_info.task_table_count_; + LOG_WARN("failed to gather table stats", K(ret)); } } } - } + offset += slice_cnt; + } while (OB_SUCC(ret) && table_ids.count() == slice_cnt); } return ret; } @@ -5520,6 +5570,9 @@ int ObDbmsStats::do_gather_table_stats(sql::ObExecContext &ctx, } else if (OB_ISNULL(table_schema)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret)); + } else if (is_recyclebin_database_id(table_schema->get_database_id()) || + (lib::is_oracle_mode() && is_oceanbase_sys_database_id(table_schema->get_database_id()))) { + //do nothing } else { StatTable stat_table(table_schema->get_database_id(), table_id); double stale_percent_threshold = OPT_DEFAULT_STALE_PERCENT; @@ -5718,6 +5771,8 @@ int ObDbmsStats::gather_table_stats_with_default_param(ObExecContext &ctx, LOG_WARN("failed to use default gather stat optitions", K(ret)); } else if (OB_FAIL(adjust_auto_gather_stat_option(stat_table.partition_stat_infos_, stat_param))) { LOG_WARN("failed to use default gather stat optitions", K(ret)); + } else if (!stat_param.need_gather_stats()) { + //do nothing } else if (OB_FAIL(running_monitor.add_table_info(stat_param, stat_table.stale_percent_))) { LOG_WARN("failed to add table info", K(ret)); } else if (OB_FAIL(ObDbmsStatsExecutor::gather_table_stats(ctx, stat_param, running_monitor))) { @@ -5751,7 +5806,9 @@ int ObDbmsStats::gather_table_stats_with_default_param(ObExecContext &ctx, LOG_TRACE("Succeed to gather table stats", K(stat_param)); } running_monitor.set_monitor_result(ret, ObTimeUtility::current_time(), stat_param.allocator_->used()); - update_optimizer_gather_stat_info(NULL, &gather_stat); + if (stat_param.need_gather_stats()) { + update_optimizer_gather_stat_info(NULL, &gather_stat); + } ObOptStatGatherStatList::instance().remove(gather_stat); task_info.completed_table_count_ ++; return ret; @@ -5867,13 +5924,21 @@ int ObDbmsStats::get_new_stat_pref(ObExecContext &ctx, } else { stat_pref = tmp_pref; } + } else if (0 == opt_name.case_compare("BLOCK_SAMPLE")) { + ObBlockSamplePrefs *tmp_pref = NULL; + if (OB_FAIL(new_stat_prefs(allocator, ctx.get_my_session(), opt_value, tmp_pref))) { + LOG_WARN("failed to new stat prefs", K(ret)); + } else { + stat_pref = tmp_pref; + } } else { ret = OB_ERR_DBMS_STATS_PL; LOG_WARN("Invalid input values for pname", K(ret), K(opt_name)); LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL, "Invalid input values for pname, Only Support CASCADE |"\ - "DEGREE | ESTIMATE_PERCENT | GRANULARITY | INCREMENTAL |"\ - "INCREMENTAL_LEVEL | METHOD_OPT | NO_INVALIDATE | OPTIONS"\ - "STALE_PERCENT | ESTIMATE_BLOCK | APPROXIMATE_NDV(global prefs unique) prefs"); + "DEGREE | ESTIMATE_PERCENT | GRANULARITY | INCREMENTAL |"\ + "INCREMENTAL_LEVEL | METHOD_OPT | NO_INVALIDATE | OPTIONS |"\ + "STALE_PERCENT | ESTIMATE_BLOCK | BLOCK_SAMPLE |"\ + "APPROXIMATE_NDV(global prefs unique) prefs"); } return ret; } @@ -6341,6 +6406,8 @@ int ObDbmsStats::gather_system_stats(sql::ObExecContext &ctx, LOG_WARN("failed to check is unix connection", K(ret)); } else if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_FAIL(check_system_stat_table_ready(session->get_effective_tenant_id()))) { LOG_WARN("failed to check system stat table ready", K(ret)); } else if (OB_FAIL(ObDbmsStatsExecutor::gather_system_stats(ctx, session->get_effective_tenant_id()))) { @@ -6373,6 +6440,8 @@ int ObDbmsStats::delete_system_stats(sql::ObExecContext &ctx, LOG_WARN("failed to check is unix connection", K(ret)); } else if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_FAIL(check_system_stat_table_ready(session->get_effective_tenant_id()))) { LOG_WARN("failed to check system stat table ready", K(ret)); } else if (OB_FAIL(ObDbmsStatsExecutor::delete_system_stats(ctx, session->get_effective_tenant_id()))) { @@ -6411,6 +6480,8 @@ int ObDbmsStats::set_system_stats(sql::ObExecContext &ctx, LOG_WARN("failed to check is unix connection", K(ret)); } else if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_FAIL(check_system_stat_table_ready(session->get_effective_tenant_id()))) { LOG_WARN("failed to check system stat table ready", K(ret)); } else if (2 != params.count()) { @@ -6544,6 +6615,7 @@ int ObDbmsStats::check_system_stat_table_ready(int64_t tenant_id) * @param result * @return int */ + int ObDbmsStats::copy_table_stats(sql::ObExecContext &ctx, sql::ParamStore ¶ms, common::ObObj &result) @@ -6560,6 +6632,8 @@ int ObDbmsStats::copy_table_stats(sql::ObExecContext &ctx, dummy_part_name.set_null(); if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_2_2_0) { //do nothing } else if (OB_FAIL(parse_table_part_info(ctx, @@ -6729,11 +6803,18 @@ int ObDbmsStats::adjust_auto_gather_stat_option(const ObIArray(COMPACTION_EVENT_MAX) == ARRAYSIZEOF(CompactionEventStr), "events str len is mismatch"); - const char *str = ""; - if (event >= COMPACTION_EVENT_MAX || event < PREPARE_UNFINISH_TABLE_IDS) { - str = "invalid_type"; - } else { - str = CompactionEventStr[event]; - } - return str; -} - -int64_t ObRSCompactionTimeGuard::to_string(char *buf, const int64_t buf_len) const -{ - int64_t pos = 0; - for (int64_t idx = 0; idx < idx_; ++idx) { - fmt_ts_to_meaningful_str(buf, buf_len, pos, get_comp_event_str((CompactionEvent)line_array_[idx]), click_poinsts_[idx]); - } - return pos; -} /** * -------------------------------------------------------------------ObTabletLSPairCache------------------------------------------------------------------- diff --git a/src/rootserver/freeze/ob_major_merge_progress_util.h b/src/rootserver/freeze/ob_major_merge_progress_util.h index 59ef668c6e..33de0a9842 100644 --- a/src/rootserver/freeze/ob_major_merge_progress_util.h +++ b/src/rootserver/freeze/ob_major_merge_progress_util.h @@ -209,26 +209,6 @@ typedef hash::ObHashMap ObTabletStatusMap; typedef common::ObArray ObTabletLSPairArray; typedef hash::ObHashMap ObTableCompactionInfoMap; -struct ObRSCompactionTimeGuard : public ObCompactionTimeGuard -{ -public: - ObRSCompactionTimeGuard() - : ObCompactionTimeGuard(UINT64_MAX, "[RS] ") - {} - virtual ~ObRSCompactionTimeGuard() {} - enum CompactionEvent : uint16_t { - PREPARE_UNFINISH_TABLE_IDS = 0, - GET_TABLET_LS_PAIRS, - GET_TABLET_META_TABLE, - CKM_VERIFICATION, - COMPACTION_EVENT_MAX, - }; - virtual int64_t to_string(char *buf, const int64_t buf_len) const override; -private: - const static char *CompactionEventStr[]; - static const char *get_comp_event_str(enum CompactionEvent event); -}; - struct ObCkmValidatorStatistics { ObCkmValidatorStatistics() { reset(); } diff --git a/src/rootserver/freeze/ob_major_merge_scheduler.cpp b/src/rootserver/freeze/ob_major_merge_scheduler.cpp index 0067cb3d88..129d6594ad 100644 --- a/src/rootserver/freeze/ob_major_merge_scheduler.cpp +++ b/src/rootserver/freeze/ob_major_merge_scheduler.cpp @@ -835,9 +835,23 @@ void ObMajorMergeScheduler::check_merge_interval_time(const bool is_merging) (GCONF.enable_major_freeze) && (!tenant_config->major_freeze_duty_time.disable())) { if (TC_REACH_TIME_INTERVAL(30 * 60 * 1000 * 1000)) { - LOG_ERROR("long time no major freeze, please check it", KR(ret), - K(global_last_merged_time), K(global_merge_start_time), K(max_merge_time), - K(now), K_(tenant_id), K(is_merging), K(start_service_time), K(total_service_time)); + // standby tenant cannot launch major freeze itself, it perform major freeze according + // to freeze info synchronized from primary tenant. therefore, standby tenants that + // stop sync from primary tenant will not perform major freeze any more. do not print + // error log for this case. issue-id: 56800988 + ObAllTenantInfo tenant_info; + if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id_, sql_proxy_, + false, tenant_info))) { + LOG_WARN("fail to load tenant info", KR(ret), K_(tenant_id)); + } else if (tenant_info.is_standby() + && (tenant_info.get_standby_scn() >= tenant_info.get_recovery_until_scn())) { + LOG_INFO("standby tenant do not sync from primary tenant any more, and do not" + " major freeze any more"); + } else { + LOG_ERROR("long time no major freeze, please check it", KR(ret), + K(global_last_merged_time), K(global_merge_start_time), K(max_merge_time), + K(now), K_(tenant_id), K(is_merging), K(start_service_time), K(total_service_time)); + } } } } diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index ed7d74e2ee..04ce0019f4 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -16672,6 +16672,10 @@ int ObDDLService::prepare_hidden_table_schema(const ObTableSchema &orig_table_sc hidden_table_schema.set_association_table_id(orig_table_schema.get_table_id()); // set the hidden attributes of the table hidden_table_schema.set_table_state_flag(ObTableStateFlag::TABLE_STATE_HIDDEN_OFFLINE_DDL); + if (orig_table_schema.get_tenant_id() != hidden_table_schema.get_tenant_id()) { + // recover restore table, do not sync log to cdc. + hidden_table_schema.set_ddl_ignore_sync_cdc_flag(ObDDLIgnoreSyncCdcFlag::DONT_SYNC_LOG_FOR_CDC); + } // in oracle mode, need to add primary key constraints if (is_oracle_mode && !hidden_table_schema.is_heap_table()) { uint64_t new_cst_id = OB_INVALID_ID; @@ -19514,6 +19518,7 @@ int ObDDLService::make_recover_restore_tables_visible(obrpc::ObAlterTableArg &al tmp_schema.set_association_table_id(OB_INVALID_ID); tmp_schema.set_table_state_flag(ObTableStateFlag::TABLE_STATE_NORMAL); tmp_schema.set_in_offline_ddl_white_list(true); + tmp_schema.set_ddl_ignore_sync_cdc_flag(ObDDLIgnoreSyncCdcFlag::DO_SYNC_LOG_FOR_CDC); // reset. ObArray conflict_schema_types; uint64_t synonym_id = OB_INVALID_ID; bool object_exist = false; diff --git a/src/rootserver/ob_ls_recovery_stat_handler.cpp b/src/rootserver/ob_ls_recovery_stat_handler.cpp index fbf2458e41..0d52fe8b17 100755 --- a/src/rootserver/ob_ls_recovery_stat_handler.cpp +++ b/src/rootserver/ob_ls_recovery_stat_handler.cpp @@ -353,6 +353,8 @@ int ObLSRecoveryStatHandler::try_reload_and_fix_config_version_(const palf::LogC share::SCN readable_scn; const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_); uint64_t tenant_data_version = 0; + ObLSRecoveryStatOperator op; + ObLSID ls_id; if (OB_FAIL(check_inner_stat_())) { LOG_WARN("inner stat error", KR(ret)); } else if (OB_UNLIKELY(!current_version.is_valid())) { @@ -361,9 +363,14 @@ int ObLSRecoveryStatHandler::try_reload_and_fix_config_version_(const palf::LogC } else if (OB_ISNULL(GCTX.sql_proxy_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("sql proxy is null", KR(ret)); + } else if (FALSE_IT(ls_id = ls_->get_ls_id())) { + //can not be there } else if (OB_FAIL(GET_MIN_DATA_VERSION(meta_tenant_id, tenant_data_version))) { LOG_WARN("failed to get min data version", KR(ret), K(tenant_id_), K(meta_tenant_id)); - } else if (tenant_data_version < DATA_VERSION_4_3_0_0) { + } else if (tenant_data_version < MOCK_DATA_VERSION_4_2_4_0) { + //内部表config_version的汇报最开始是在4300版本上提交的 + //后面patch到424版本上，由于是在4300分支的第一个版本号提交 + //所以版本号判断直接小于等于424即可 need_update = false; LOG_INFO("not ready to load and update config version", KR(ret), K(tenant_data_version)); } else { @@ -371,13 +378,10 @@ int ObLSRecoveryStatHandler::try_reload_and_fix_config_version_(const palf::LogC if (current_version != config_version_in_inner_) { need_update = true; FLOG_INFO("config version not match, need update", - K(config_version_in_inner_), K(current_version), "ls_id", - ls_->get_ls_id()); + K(config_version_in_inner_), K(current_version), K(ls_id)); } } if (OB_SUCC(ret) && need_update) { - ObLSRecoveryStatOperator op; - ObLSID ls_id = ls_->get_ls_id(); if (OB_FAIL(op.update_ls_config_version(tenant_id_, ls_id, current_version, *GCTX.sql_proxy_, readable_scn))) { LOG_WARN("failed to update ls config version", KR(ret), K(tenant_id_), K(ls_id), K(current_version)); diff --git a/src/rootserver/ob_recovery_ls_service.cpp b/src/rootserver/ob_recovery_ls_service.cpp index 76058da1a1..f122f77e0b 100755 --- a/src/rootserver/ob_recovery_ls_service.cpp +++ b/src/rootserver/ob_recovery_ls_service.cpp @@ -474,7 +474,7 @@ int ObRecoveryLSService::process_ls_tx_log_(ObTxLogBlock &tx_log_block, const SC const ObTxBufferNodeArray &source_data = commit_log.get_multi_source_data(); const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id_); - START_TRANSACTION(proxy_, exec_tenant_id) + ObMySQLTransaction trans; for (int64_t i = 0; OB_SUCC(ret) && i < source_data.count(); ++i) { const ObTxBufferNode &node = source_data.at(i); if (ObTxDataSourceType::STANDBY_UPGRADE == node.get_data_source_type()) { @@ -484,6 +484,8 @@ int ObRecoveryLSService::process_ls_tx_log_(ObTxLogBlock &tx_log_block, const SC } else if (ObTxDataSourceType::LS_TABLE != node.get_data_source_type() && ObTxDataSourceType::TRANSFER_TASK != node.get_data_source_type()) { // nothing + } else if (! trans.is_started() && OB_FAIL(trans.start(proxy_, exec_tenant_id))) { + LOG_WARN("failed to start trans", KR(ret), K(exec_tenant_id)); } else if (FALSE_IT(has_operation = true)) { //can not be there; } else if (OB_FAIL(check_valid_to_operator_ls_(sync_scn))) { diff --git a/src/share/allocator/ob_shared_memory_allocator_mgr.h b/src/share/allocator/ob_shared_memory_allocator_mgr.h index df57da87bd..554ad3b763 100644 --- a/src/share/allocator/ob_shared_memory_allocator_mgr.h +++ b/src/share/allocator/ob_shared_memory_allocator_mgr.h @@ -44,6 +44,8 @@ public: SHARE_LOG(ERROR, "init memstore allocator failed", KR(ret)); } else if (OB_FAIL(mds_allocator_.init())) { SHARE_LOG(ERROR, "init mds allocator failed", KR(ret)); + } else if (OB_FAIL(tx_data_op_allocator_.init())) { + SHARE_LOG(ERROR, "init tx data op allocator failed", KR(ret)); } else if (OB_FAIL( share_resource_throttle_tool_.init(&memstore_allocator_, &tx_data_allocator_, &mds_allocator_))) { SHARE_LOG(ERROR, "init share resource throttle tool failed", KR(ret)); @@ -65,6 +67,7 @@ public: ObTenantTxDataAllocator &tx_data_allocator() { return tx_data_allocator_; } ObTenantMdsAllocator &mds_allocator() { return mds_allocator_; } TxShareThrottleTool &share_resource_throttle_tool() { return share_resource_throttle_tool_; } + ObTenantTxDataOpAllocator &tx_data_op_allocator() { return tx_data_op_allocator_; } private: void update_share_throttle_config_(const int64_t total_memory, omt::ObTenantConfigGuard &config); @@ -78,6 +81,7 @@ private: ObMemstoreAllocator memstore_allocator_; ObTenantTxDataAllocator tx_data_allocator_; ObTenantMdsAllocator mds_allocator_; + ObTenantTxDataOpAllocator tx_data_op_allocator_; }; class TxShareMemThrottleUtil @@ -156,4 +160,4 @@ public: } // namespace share } // namespace oceanbase -#endif \ No newline at end of file +#endif diff --git a/src/share/allocator/ob_tx_data_allocator.cpp b/src/share/allocator/ob_tx_data_allocator.cpp index a14c76f44d..a1f10d051c 100644 --- a/src/share/allocator/ob_tx_data_allocator.cpp +++ b/src/share/allocator/ob_tx_data_allocator.cpp @@ -23,6 +23,8 @@ namespace oceanbase { namespace share { +thread_local int64_t ObTenantTxDataOpAllocator::local_alloc_size_ = 0; + int64_t ObTenantTxDataAllocator::resource_unit_size() { static const int64_t TX_DATA_RESOURCE_UNIT_SIZE = OB_MALLOC_NORMAL_BLOCK_SIZE; /* 8KB */ @@ -149,5 +151,59 @@ ObTxDataThrottleGuard::~ObTxDataThrottleGuard() } } +int ObTenantTxDataOpAllocator::init() +{ + int ret = OB_SUCCESS; + ObMemAttr mem_attr; + mem_attr.tenant_id_ = MTL_ID(); + mem_attr.ctx_id_ = ObCtxIds::MDS_DATA_ID; + mem_attr.label_ = "TX_OP"; + ObSharedMemAllocMgr *share_mem_alloc_mgr = MTL(ObSharedMemAllocMgr *); + throttle_tool_ = &(share_mem_alloc_mgr->share_resource_throttle_tool()); + if (IS_INIT){ + ret = OB_INIT_TWICE; + SHARE_LOG(WARN, "init tenant mds allocator twice", KR(ret), KPC(this)); + } else if (OB_ISNULL(throttle_tool_)) { + ret = OB_ERR_UNEXPECTED; + SHARE_LOG(WARN, "throttle tool is unexpected null", KP(throttle_tool_), KP(share_mem_alloc_mgr)); + } else if (OB_FAIL(allocator_.init(OB_MALLOC_NORMAL_BLOCK_SIZE, block_alloc_, mem_attr))) { + MDS_LOG(WARN, "init vslice allocator failed", K(ret), K(OB_MALLOC_NORMAL_BLOCK_SIZE), KP(this), K(mem_attr)); + } else { + allocator_.set_nway(MDS_ALLOC_CONCURRENCY); + is_inited_ = true; + } + return ret; +} + +void *ObTenantTxDataOpAllocator::alloc(const int64_t size) +{ + int64_t abs_expire_time = THIS_WORKER.get_timeout_ts(); + void * buf = alloc(size, abs_expire_time); + if (OB_NOT_NULL(buf)) { + local_alloc_size_ += size; + } + return buf; +} + +void *ObTenantTxDataOpAllocator::alloc(const int64_t size, const ObMemAttr &attr) +{ + UNUSED(attr); + void *obj = alloc(size); + return obj; +} + +void *ObTenantTxDataOpAllocator::alloc(const int64_t size, const int64_t abs_expire_time) +{ + void *obj = allocator_.alloc(size); + return obj; +} + +void ObTenantTxDataOpAllocator::free(void *ptr) +{ + allocator_.free(ptr); +} + +void ObTenantTxDataOpAllocator::set_attr(const ObMemAttr &attr) { allocator_.set_attr(attr); } + } // namespace share -} // namespace oceanbase \ No newline at end of file +} // namespace oceanbase diff --git a/src/share/allocator/ob_tx_data_allocator.h b/src/share/allocator/ob_tx_data_allocator.h index ab146549f4..56c3bf8cbd 100644 --- a/src/share/allocator/ob_tx_data_allocator.h +++ b/src/share/allocator/ob_tx_data_allocator.h @@ -16,6 +16,7 @@ #include "lib/allocator/ob_slice_alloc.h" #include "share/ob_delegate.h" #include "share/throttle/ob_share_throttle_define.h" +#include "lib/allocator/ob_vslice_alloc.h" namespace oceanbase { namespace share { @@ -75,7 +76,35 @@ private: share::TxShareThrottleTool *throttle_tool_; }; +class ObTenantTxDataOpAllocator : public ObIAllocator { +private: + static const int64_t MDS_ALLOC_CONCURRENCY = 32; +public: + DEFINE_CUSTOM_FUNC_FOR_THROTTLE(Mds); + +public: + ObTenantTxDataOpAllocator() : is_inited_(false), throttle_tool_(nullptr), block_alloc_(), allocator_() {} + + int init(); + void destroy() { is_inited_ = false; } + void *alloc(const int64_t size, const int64_t expire_ts); + virtual void *alloc(const int64_t size) override; + virtual void *alloc(const int64_t size, const ObMemAttr &attr) override; + virtual void free(void *ptr) override; + virtual void set_attr(const ObMemAttr &attr) override; + int64_t hold() { return allocator_.hold(); } + int64_t get_local_alloc_size() { return local_alloc_size_; } + void reset_local_alloc_size() { local_alloc_size_ = 0; } + TO_STRING_KV(K(is_inited_), KP(this), KP(throttle_tool_), KP(&block_alloc_), KP(&allocator_)); + +private: + bool is_inited_; + share::TxShareThrottleTool *throttle_tool_; + common::ObBlockAllocMgr block_alloc_; + common::ObVSliceAlloc allocator_; + static thread_local int64_t local_alloc_size_; +}; } // namespace share } // namespace oceanbase -#endif \ No newline at end of file +#endif diff --git a/src/share/compaction/ob_compaction_time_guard.cpp b/src/share/compaction/ob_compaction_time_guard.cpp index 156e1647bc..348ccb3102 100644 --- a/src/share/compaction/ob_compaction_time_guard.cpp +++ b/src/share/compaction/ob_compaction_time_guard.cpp @@ -14,45 +14,263 @@ namespace oceanbase namespace compaction { +/** + * -------------------------------------------------------------------ObCompactionTimeGuard------------------------------------------------------------------- + */ +ObCompactionTimeGuard::~ObCompactionTimeGuard() +{ + int64_t total_cost = 0; + for (int64_t idx = 0; idx < size_; ++idx) { + total_cost += event_times_[idx]; + } + total_cost += common::ObTimeUtility::current_time() - last_click_ts_; + if (OB_UNLIKELY(total_cost >= warn_threshold_)) { + ::oceanbase::common::OB_PRINT(log_mod_, OB_LOG_LEVEL_DIRECT_NO_ERRCODE(WARN), OB_SUCCESS, "cost too much time", LOG_KVS(K(*this))); + } +} + +int64_t ObCompactionTimeGuard::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + fmt_ts_to_meaningful_str(buf, buf_len, pos, "|threshold", warn_threshold_); + common::databuff_printf(buf, buf_len, pos, "start at %s|", common::ObTime2Str::ob_timestamp_str(add_time_)); + int64_t total_cost = 0; + for (int64_t idx = 0; idx < size_; ++idx) { + const uint64_t ts = event_times_[idx]; + if (ts < 1_ms) { + common::databuff_printf(buf, buf_len, pos, "%ldus|", ts); + } else if (ts < 1_s) { + common::databuff_printf(buf, buf_len, pos, "%.2lfms|", double(ts) / 1_ms); + } else { + common::databuff_printf(buf, buf_len, pos, "%.2lfs|", double(ts) / 1_s); + } + total_cost += event_times_[idx]; + } + total_cost += common::ObTimeUtility::current_time() - last_click_ts_; + fmt_ts_to_meaningful_str(buf, buf_len, pos, "total", total_cost); + if (pos != 0 && pos < buf_len) { + pos -= 1; + } + return pos; +} + +void ObCompactionTimeGuard::reuse() +{ + size_ = 0; + last_click_ts_ = common::ObTimeUtility::current_time(); + add_time_ = common::ObTimeUtility::current_time(); + for (uint16_t i = 0; i < capacity_; ++i) { + event_times_[i] = 0; + } +} + +bool ObCompactionTimeGuard::click(const uint16_t event) +{ + if (OB_LIKELY(event < CAPACITY)) { + if (OB_LIKELY(size_ <= event)) { + size_ = event + 1; + } + const int64_t now = common::ObTimeUtility::current_time(); + event_times_[event] += now - last_click_ts_; + last_click_ts_ = now; + } + return true; +} + +void ObCompactionTimeGuard::fmt_ts_to_meaningful_str( + char *buf, + const int64_t buf_len, + int64_t &pos, + const char *lvalue, + const int64_t ts) const +{ + common::databuff_printf(buf, buf_len, pos, "%s", lvalue); + if (ts < 1_ms) { + common::databuff_printf(buf, buf_len, pos, "=%ldus|", ts); + } else if (ts < 1_s) { + common::databuff_printf(buf, buf_len, pos, "=%.2lfms|", double(ts) / 1_ms); + } else { + common::databuff_printf(buf, buf_len, pos, "=%.2lfs|", double(ts) / 1_s); + } +} void ObCompactionTimeGuard::add_time_guard(const ObCompactionTimeGuard &other) { - // last_click_ts_ is not useflu - ObCompactionTimeGuard time_guard; - int i = 0; - int j = 0; - while (i < idx_ && j < other.idx_) { - if (line_array_[i] == other.line_array_[j]) { - time_guard.line_array_[time_guard.idx_] = line_array_[i]; - time_guard.click_poinsts_[time_guard.idx_++] = click_poinsts_[i++] + other.click_poinsts_[j++]; - } else if (line_array_[i] < other.line_array_[j]) { - time_guard.line_array_[time_guard.idx_] = line_array_[i]; - time_guard.click_poinsts_[time_guard.idx_++] = click_poinsts_[i++]; - } else { - time_guard.line_array_[time_guard.idx_] = other.line_array_[j]; - time_guard.click_poinsts_[time_guard.idx_++] = other.click_poinsts_[j++]; + if (OB_LIKELY(guard_type_ == other.guard_type_ && CAPACITY == other.capacity_)) { + size_ = std::max(size_, other.size_); + for (uint16_t i = 0; i < size_; i++) { + event_times_[i] += other.event_times_[i]; } } - while (i < idx_) { - time_guard.line_array_[time_guard.idx_] = line_array_[i]; - time_guard.click_poinsts_[time_guard.idx_++] = click_poinsts_[i++]; - } - while (j < other.idx_) { - time_guard.line_array_[time_guard.idx_] = other.line_array_[j]; - time_guard.click_poinsts_[time_guard.idx_++] = other.click_poinsts_[j++]; - } - *this = time_guard; } ObCompactionTimeGuard & ObCompactionTimeGuard::operator=(const ObCompactionTimeGuard &other) { + guard_type_ = other.guard_type_; + capacity_ = other.capacity_; + size_ = other.size_; last_click_ts_ = other.last_click_ts_; - idx_ = other.idx_; - for (int i = 0; i < other.idx_; ++i) { - line_array_[i] = other.line_array_[i]; - click_poinsts_[i] = other.click_poinsts_[i]; + add_time_ = other.add_time_; + for (uint16_t i = 0; i < other.size_; ++i) { + event_times_[i] = other.event_times_[i]; } return *this; } +uint16_t ObCompactionTimeGuard::get_max_event_count(const ObCompactionTimeGuardType guard_type) +{ + uint16_t max_event_count = CAPACITY; + if (RS_COMPACT_TIME_GUARD == guard_type) { + max_event_count = ObRSCompactionTimeGuard::COMPACTION_EVENT_MAX; + } else if (SCHEDULE_COMPACT_TIME_GUARD == guard_type) { + max_event_count = ObCompactionScheduleTimeGuard::COMPACTION_EVENT_MAX; + } else if (STORAGE_COMPACT_TIME_GUARD == guard_type) { + max_event_count = ObStorageCompactionTimeGuard::COMPACTION_EVENT_MAX; + } + return max_event_count; +} + +/** + * -------------------------------------------------------------------ObRSCompactionTimeGuard------------------------------------------------------------------- + */ +const char *ObRSCompactionTimeGuard::CompactionEventStr[] = { + "PREPARE_UNFINISH_TABLE_IDS", + "GET_TABLET_LS_PAIRS", + "GET_TABLET_META_TABLE", + "CKM_VERIFICATION" +}; + +const char *ObRSCompactionTimeGuard::get_comp_event_str(enum CompactionEvent event) +{ + STATIC_ASSERT(static_cast(COMPACTION_EVENT_MAX) == ARRAYSIZEOF(CompactionEventStr), "events str len is mismatch"); + STATIC_ASSERT(static_cast(COMPACTION_EVENT_MAX) <= static_cast(CAPACITY), "too many events, need update CAPACITY"); + const char *str = ""; + if (event >= COMPACTION_EVENT_MAX || event < PREPARE_UNFINISH_TABLE_IDS) { + str = "invalid_type"; + } else { + str = CompactionEventStr[event]; + } + return str; +} + +int64_t ObRSCompactionTimeGuard::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + for (uint16_t idx = 0; idx < size_; ++idx) { + if (event_times_[idx] > 0) { + fmt_ts_to_meaningful_str(buf, buf_len, pos, get_comp_event_str(static_cast(idx)), event_times_[idx]); + } + } + return pos; +} + +/** + * ObCompactionScheduleTimeGuard Impl + */ +const char *ObCompactionScheduleTimeGuard::CompactionEventStr[] = { + "GET_TABLET", + "UPDATE_TABLET_REPORT_STATUS", + "READ_MEDIUM_INFO", + "SCHEDULE_NEXT_MEDIUM", + "SCHEDULE_TABLET_MEDIUM", + "FAST_FREEZE", + "SEARCH_META_TABLE", + "CHECK_META_TABLE", + "SEARCH_CHECKSUM", + "CHECK_CHECKSUM", + "SCHEDULER_NEXT_ROUND" +}; + +const char *ObCompactionScheduleTimeGuard::get_comp_event_str(enum CompactionEvent event) +{ + STATIC_ASSERT(static_cast(COMPACTION_EVENT_MAX) == ARRAYSIZEOF(CompactionEventStr), "events str len is mismatch"); + STATIC_ASSERT(static_cast(COMPACTION_EVENT_MAX) <= static_cast(CAPACITY), "too many events, need update CAPACITY"); + const char *str = ""; + if (event >= COMPACTION_EVENT_MAX || event < GET_TABLET) { + str = "invalid_type"; + } else { + str = CompactionEventStr[event]; + } + return str; +} + +int64_t ObCompactionScheduleTimeGuard::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + for (int16_t idx = 0; idx < size_; ++idx) { + if (event_times_[idx] > 0) { + fmt_ts_to_meaningful_str(buf, buf_len, pos, get_comp_event_str(static_cast(idx)), event_times_[idx]); + } + } + return pos; +} + +/* + * ----------------------------------------------ObCompactionTimeGuard-------------------------------------------------- + */ +constexpr float ObStorageCompactionTimeGuard::COMPACTION_SHOW_PERCENT_THRESHOLD; +const char *ObStorageCompactionTimeGuard::CompactionEventStr[] = { + "WAIT_TO_SCHEDULE", + "COMPACTION_POLICY", + "PRE_PROCESS_TX_TABLE", + "GET_PARALLEL_RANGE", + "EXECUTE", + "CREATE_SSTABLE", + "UPDATE_TABLET", + "RELEASE_MEMTABLE", + "SCHEDULE_OTHER_COMPACTION", + "DAG_FINISH" +}; + +const char *ObStorageCompactionTimeGuard::get_comp_event_str(const enum CompactionEvent event) +{ + STATIC_ASSERT(static_cast(COMPACTION_EVENT_MAX) == ARRAYSIZEOF(CompactionEventStr), "events str len is mismatch"); + STATIC_ASSERT(static_cast(COMPACTION_EVENT_MAX) <= static_cast(CAPACITY), "too many events, need update CAPACITY"); + const char *str = ""; + if (event >= COMPACTION_EVENT_MAX || event < DAG_WAIT_TO_SCHEDULE) { + str = "invalid_type"; + } else { + str = CompactionEventStr[event]; + } + return str; +} + +int64_t ObStorageCompactionTimeGuard::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + int64_t total_cost = 0; + J_KV(K_(add_time)); + common::databuff_printf(buf, buf_len, pos, "|"); + if (size_ > DAG_WAIT_TO_SCHEDULE && event_times_[DAG_WAIT_TO_SCHEDULE] > COMPACTION_SHOW_TIME_THRESHOLD) { + fmt_ts_to_meaningful_str(buf, buf_len, pos, "wait_schedule_time", event_times_[DAG_WAIT_TO_SCHEDULE]); + } + for (int64_t idx = COMPACTION_POLICY; idx < size_; ++idx) { + total_cost += event_times_[idx]; + } + if (total_cost > COMPACTION_SHOW_TIME_THRESHOLD) { + float ratio = 0; + for (int64_t idx = COMPACTION_POLICY; idx < size_; ++idx) { + const uint32_t time_interval = event_times_[idx]; // include the retry time since previous event + ratio = (float)(time_interval)/ total_cost; + if (ratio >= COMPACTION_SHOW_PERCENT_THRESHOLD || time_interval >= COMPACTION_SHOW_TIME_THRESHOLD) { + fmt_ts_to_meaningful_str(buf, buf_len, pos, get_comp_event_str(static_cast(idx)), event_times_[idx]); + if (ratio > 0.01) { + common::databuff_printf(buf, buf_len, pos, "(%.2f)", ratio); + } + common::databuff_printf(buf, buf_len, pos, "|"); + } + } + } + fmt_ts_to_meaningful_str(buf, buf_len, pos, "total", total_cost); + if (pos != 0 && pos < buf_len) { + buf[pos - 1] = ';'; + } + + if (pos != 0 && pos < buf_len) { + pos -= 1; + } + return pos; +} + + } // namespace compaction } // namespace oceanbase diff --git a/src/share/compaction/ob_compaction_time_guard.h b/src/share/compaction/ob_compaction_time_guard.h index dd17ddbae9..c5253404a7 100644 --- a/src/share/compaction/ob_compaction_time_guard.h +++ b/src/share/compaction/ob_compaction_time_guard.h @@ -10,48 +10,169 @@ #ifndef OB_SHARE_COMPACTION_COMPACTION_TIME_GUARD_H_ #define OB_SHARE_COMPACTION_COMPACTION_TIME_GUARD_H_ #include "share/ob_occam_time_guard.h" +#include "lib/container/ob_se_array.h" namespace oceanbase { namespace compaction { -class ObCompactionTimeGuard : public common::occam::ObOccamTimeGuard +/* + * ObCompactionTimeGuard refers to the implementation of from ObOccamTimeGuard. + * For example, you have 3 enum events {e0, e1, e2} + * If you want to record the time cost of event e1, you can use click(e1), then event_times_[1] will accumulate the time cost of e1. + * + * ObCompactionTimeGuard + * -- ObRSCompactionTimeGuard + * -- ObScheduleCompactionTimeGuard + * -- ObStorageCompactionTimeGuard + */ +class ObCompactionTimeGuard { public: - const static uint64_t WARN_THRESHOLD = 30L * 1000 * 1000; // 30s - ObCompactionTimeGuard(const uint64_t warn_threshold = WARN_THRESHOLD, const char *mod = "") - : ObOccamTimeGuard(warn_threshold, nullptr, nullptr, mod), - add_time_(0) - {} - virtual ~ObCompactionTimeGuard() {} - virtual int64_t to_string(char *buf, const int64_t buf_len) const + enum ObCompactionTimeGuardType : uint8_t { - UNUSEDx(buf, buf_len); - return 0; + BASE_COMPACT_TIME_GUARD = 0, + RS_COMPACT_TIME_GUARD = 1, + SCHEDULE_COMPACT_TIME_GUARD = 2, + STORAGE_COMPACT_TIME_GUARD = 3, + MAX_COMPACT_TIME_GUARD + }; +public: + const static uint64_t WARN_THRESHOLD = 30L * 1000 * 1000; // 30s + const static uint16_t CAPACITY = 16; + ObCompactionTimeGuard(const ObCompactionTimeGuardType gurad_type = BASE_COMPACT_TIME_GUARD, + const uint64_t warn_threshold = WARN_THRESHOLD, + const char *mod = "") + : guard_type_(gurad_type), + warn_threshold_(warn_threshold), + log_mod_(mod), + capacity_(get_max_event_count(gurad_type)), + size_(0), + last_click_ts_(common::ObTimeUtility::current_time()), + add_time_(common::ObTimeUtility::current_time()) + { + reuse(); } + virtual ~ObCompactionTimeGuard(); + virtual int64_t to_string(char *buf, const int64_t buf_len) const; + void reuse(); void add_time_guard(const ObCompactionTimeGuard &other); ObCompactionTimeGuard & operator=(const ObCompactionTimeGuard &other); - OB_INLINE bool is_empty() const { return 0 == idx_; } + OB_INLINE bool is_empty() const { return 0 == size_; } // set the dag add_time as the first click time OB_INLINE void set_last_click_ts(const int64_t time) { last_click_ts_ = time; add_time_ = time; } - OB_INLINE uint32_t get_specified_cost_time(const int64_t line) const { + OB_INLINE uint32_t get_specified_cost_time(const int64_t event) const { uint32_t ret_val = 0; - for (int64_t idx = 0; idx < idx_; ++idx) { - if (line_array_[idx] == line) { - ret_val = click_poinsts_[idx]; - break; - } + if (OB_LIKELY(event < size_)) { + ret_val = event_times_[event]; } return ret_val; } - + bool click(const uint16_t event); + // copy from ObOccamTimeGuard + void fmt_ts_to_meaningful_str( + char *buf, + const int64_t buf_len, + int64_t &pos, + const char *lvalue, + const int64_t ts) const; +public: + template + static constexpr uint16_t event_idx(E e) { return static_cast(e); } + static uint16_t get_max_event_count(const ObCompactionTimeGuardType guard_type); +public: + ObCompactionTimeGuardType guard_type_; + const uint64_t warn_threshold_; + const char * log_mod_; + uint16_t capacity_; // equal with CAPACITY, used for child class to check the array boundary + uint16_t size_; + int64_t last_click_ts_; int64_t add_time_; + uint64_t event_times_[CAPACITY]; }; +struct ObRSCompactionTimeGuard : public ObCompactionTimeGuard +{ +public: + ObRSCompactionTimeGuard() + : ObCompactionTimeGuard(RS_COMPACT_TIME_GUARD, UINT64_MAX, "[RS] ") + {} + virtual ~ObRSCompactionTimeGuard() {} + enum CompactionEvent : uint16_t { + PREPARE_UNFINISH_TABLE_IDS = 0, + GET_TABLET_LS_PAIRS, + GET_TABLET_META_TABLE, + CKM_VERIFICATION, + COMPACTION_EVENT_MAX, + }; + virtual int64_t to_string(char *buf, const int64_t buf_len) const override; +private: + const static char *CompactionEventStr[]; + static const char *get_comp_event_str(const enum CompactionEvent event); +}; +struct ObCompactionScheduleTimeGuard : public ObCompactionTimeGuard +{ +public: + ObCompactionScheduleTimeGuard() + : ObCompactionTimeGuard(SCHEDULE_COMPACT_TIME_GUARD, UINT64_MAX, "[STORAGE] ") + {} + virtual ~ObCompactionScheduleTimeGuard() {} + enum CompactionEvent : uint16_t { + // medium scheduler + GET_TABLET = 0, + UPDATE_TABLET_REPORT_STATUS, + READ_MEDIUM_INFO, + SCHEDULE_NEXT_MEDIUM, + SCHEDULE_TABLET_MEDIUM, + FAST_FREEZE, + // medium checker + SEARCH_META_TABLE, + CHECK_META_TABLE, + SEARCH_CHECKSUM, + CHECK_CHECKSUM, + SCHEDULER_NEXT_ROUND, + COMPACTION_EVENT_MAX + }; + virtual int64_t to_string(char *buf, const int64_t buf_len) const override; +private: + const static char *CompactionEventStr[]; + static const char *get_comp_event_str(const enum CompactionEvent event); +}; + +struct ObStorageCompactionTimeGuard : public ObCompactionTimeGuard +{ +public: + ObStorageCompactionTimeGuard() + : ObCompactionTimeGuard(STORAGE_COMPACT_TIME_GUARD, COMPACTION_WARN_THRESHOLD_RATIO, "[STORAGE] ") + {} + virtual ~ObStorageCompactionTimeGuard() {} + enum CompactionEvent : uint16_t { + DAG_WAIT_TO_SCHEDULE = 0, + COMPACTION_POLICY, + PRE_PROCESS_TX_TABLE, + GET_PARALLEL_RANGE, + EXECUTE, + CREATE_SSTABLE, + UPDATE_TABLET, + RELEASE_MEMTABLE, + SCHEDULE_OTHER_COMPACTION, + DAG_FINISH, + COMPACTION_EVENT_MAX + }; + virtual int64_t to_string(char *buf, const int64_t buf_len) const override; +private: + const static char *CompactionEventStr[]; + static const char *get_comp_event_str(const enum CompactionEvent event); + static const int64_t COMPACTION_WARN_THRESHOLD_RATIO = 60 * 1000L * 1000L; // 1 min + static constexpr float COMPACTION_SHOW_PERCENT_THRESHOLD = 0.1; + static const int64_t COMPACTION_SHOW_TIME_THRESHOLD = 1 * 1000L * 1000L; // 1s +}; + + } // namespace compaction } // namespace oceanbase diff --git a/src/share/datum/ob_datum_cmp_func_def.h b/src/share/datum/ob_datum_cmp_func_def.h index bee82eedb2..2d9d3e870b 100644 --- a/src/share/datum/ob_datum_cmp_func_def.h +++ b/src/share/datum/ob_datum_cmp_func_def.h @@ -147,7 +147,17 @@ struct ObFixedDoubleCmp: public ObDefined<> cmp_ret = 0; const double l = l_datum.get_double(); const double r = r_datum.get_double(); - if (l == r || fabs(l - r) < P) { + if (isnan(l) || isnan(r)) { + if (isnan(l) && isnan(r)) { + cmp_ret = 0; + } else if (isnan(l)) { + // l is nan, r is not nan:left always bigger than right + cmp_ret = 1; + } else { + // l is not nan, r is nan, left always less than right + cmp_ret = -1; + } + } else if (l == r || fabs(l - r) < P) { cmp_ret = 0; } else { cmp_ret = (l < r ? -1 : 1); diff --git a/src/share/inner_table/ob_inner_table_schema.21101_21150.cpp b/src/share/inner_table/ob_inner_table_schema.21101_21150.cpp index 43549a8411..7cc6dd9641 100644 --- a/src/share/inner_table/ob_inner_table_schema.21101_21150.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21101_21150.cpp @@ -60,7 +60,7 @@ int ObInnerTableSchema::gv_ob_sstables_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT M.SVR_IP, M.SVR_PORT, (case M.TABLE_TYPE when 0 then 'MEMTABLE' when 1 then 'TX_DATA_MEMTABLE' when 2 then 'TX_CTX_MEMTABLE' when 3 then 'LOCK_MEMTABLE' when 4 then 'DIRECT_LOAD_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR' when 12 then 'MINI' when 13 then 'META' when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'DDL_MEM' when 17 then 'CO_MAJOR' when 18 then 'NORMAL_CG' when 19 then 'ROWKEY_CG' when 20 then 'DDL_MERGE' else 'INVALID' end) as TABLE_TYPE, M.TENANT_ID, M.LS_ID, M.TABLET_ID, M.CG_IDX, M.START_LOG_SCN, M.END_LOG_SCN, M.DATA_CHECKSUM, M.SIZE, M.REF, M.UPPER_TRANS_VERSION, M.IS_ACTIVE, M.CONTAIN_UNCOMMITTED_ROW FROM oceanbase.__all_virtual_table_mgr M )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT M.SVR_IP, M.SVR_PORT, (case M.TABLE_TYPE when 0 then 'MEMTABLE' when 1 then 'TX_DATA_MEMTABLE' when 2 then 'TX_CTX_MEMTABLE' when 3 then 'LOCK_MEMTABLE' when 4 then 'DIRECT_LOAD_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR' when 12 then 'MINI' when 13 then 'META' when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'DDL_MEM' when 17 then 'CO_MAJOR' when 18 then 'NORMAL_CG' when 19 then 'ROWKEY_CG' when 20 then 'COL_ORIENTED_META' when 21 then 'DDL_MERGE_CO' when 22 then 'DDL_MERGE_CG' when 23 then 'DDL_MEM_CO' when 24 then 'DDL_MEM_CG' when 25 then 'DDL_MEM_MINI_SSTABLE' when 26 then 'MDS_MINI' when 27 then 'MDS_MINOR' else 'INVALID' end) as TABLE_TYPE, M.TENANT_ID, M.LS_ID, M.TABLET_ID, M.CG_IDX, M.START_LOG_SCN, M.END_LOG_SCN, M.DATA_CHECKSUM, M.SIZE, M.REF, M.UPPER_TRANS_VERSION, M.IS_ACTIVE, M.CONTAIN_UNCOMMITTED_ROW FROM oceanbase.__all_virtual_table_mgr M )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.28051_28100.cpp b/src/share/inner_table/ob_inner_table_schema.28051_28100.cpp index 4eee2968e5..ebb30f004e 100644 --- a/src/share/inner_table/ob_inner_table_schema.28051_28100.cpp +++ b/src/share/inner_table/ob_inner_table_schema.28051_28100.cpp @@ -310,7 +310,7 @@ int ObInnerTableSchema::gv_ob_sstables_ora_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT M.SVR_IP, M.SVR_PORT, (case M.TABLE_TYPE when 0 then 'MEMTABLE' when 1 then 'TX_DATA_MEMTABLE' when 2 then 'TX_CTX_MEMTABLE' when 3 then 'LOCK_MEMTABLE' when 4 then 'DIRECT_LOAD_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR' when 12 then 'MINI' when 13 then 'META' when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'IMC_SEGMENT' when 20 then 'DDL_MERGE' else 'INVALID' end) as TABLE_TYPE, M.LS_ID, M.TABLET_ID, M.START_LOG_SCN, M.END_LOG_SCN, M."SIZE", M.REF, M.UPPER_TRANS_VERSION, M.IS_ACTIVE, M.CONTAIN_UNCOMMITTED_ROW FROM SYS.ALL_VIRTUAL_TABLE_MGR M )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT M.SVR_IP, M.SVR_PORT, (case M.TABLE_TYPE when 0 then 'MEMTABLE' when 1 then 'TX_DATA_MEMTABLE' when 2 then 'TX_CTX_MEMTABLE' when 3 then 'LOCK_MEMTABLE' when 4 then 'DIRECT_LOAD_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR' when 12 then 'MINI' when 13 then 'META' when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'DDL_MEM' when 17 then 'CO_MAJOR' when 18 then 'NORMAL_CG' when 19 then 'ROWKEY_CG' when 20 then 'COL_ORIENTED_META' when 21 then 'DDL_MERGE_CO' when 22 then 'DDL_MERGE_CG' when 23 then 'DDL_MEM_CO' when 24 then 'DDL_MEM_CG' when 25 then 'DDL_MEM_MINI_SSTABLE' when 26 then 'MDS_MINI' when 27 then 'MDS_MINOR' else 'INVALID' end) as TABLE_TYPE, M.LS_ID, M.TABLET_ID, M.START_LOG_SCN, M.END_LOG_SCN, M."SIZE", M.REF, M.UPPER_TRANS_VERSION, M.IS_ACTIVE, M.CONTAIN_UNCOMMITTED_ROW FROM SYS.ALL_VIRTUAL_TABLE_MGR M )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index d21bb0c036..74b523d1d7 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -14382,6 +14382,8 @@ def_table_schema( ) # 12488: __all_virtual_scheduler_job_run_detail_v2 +# 12489: __all_virtual_deadlock_detector_stat +# 12490: __all_virtual_spatial_reference_systems # 余留位置（此行之前占位） # 本区域占位建议：采用真实表名进行占位 @@ -14861,6 +14863,7 @@ def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15451' def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15456', all_def_keywords['__all_virtual_nic_info']))) # 15457: __all_virtual_query_response_time # 15458: __all_scheduler_job_run_detail_v2 +# 15459: __all_spatial_reference_systems # # 余留位置（此行之前占位） # 本区域定义的Oracle表名比较复杂，一般都采用gen_xxx_table_def()方式定义，占位建议采用基表表名占位 @@ -17846,7 +17849,10 @@ SELECT when 3 then 'LOCK_MEMTABLE' when 4 then 'DIRECT_LOAD_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR' when 12 then 'MINI' when 13 then 'META' when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'DDL_MEM' - when 17 then 'CO_MAJOR' when 18 then 'NORMAL_CG' when 19 then 'ROWKEY_CG' when 20 then 'DDL_MERGE' + when 17 then 'CO_MAJOR' when 18 then 'NORMAL_CG' when 19 then 'ROWKEY_CG' when 20 then 'COL_ORIENTED_META' + when 21 then 'DDL_MERGE_CO' when 22 then 'DDL_MERGE_CG' when 23 then 'DDL_MEM_CO' + when 24 then 'DDL_MEM_CG' when 25 then 'DDL_MEM_MINI_SSTABLE' + when 26 then 'MDS_MINI' when 27 then 'MDS_MINOR' else 'INVALID' end) as TABLE_TYPE, M.TENANT_ID, @@ -56036,7 +56042,11 @@ SELECT when 0 then 'MEMTABLE' when 1 then 'TX_DATA_MEMTABLE' when 2 then 'TX_CTX_MEMTABLE' when 3 then 'LOCK_MEMTABLE' when 4 then 'DIRECT_LOAD_MEMTABLE' when 10 then 'MAJOR' when 11 then 'MINOR' when 12 then 'MINI' when 13 then 'META' - when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'IMC_SEGMENT' when 20 then 'DDL_MERGE' + when 14 then 'DDL_DUMP' when 15 then 'REMOTE_LOGICAL_MINOR' when 16 then 'DDL_MEM' + when 17 then 'CO_MAJOR' when 18 then 'NORMAL_CG' when 19 then 'ROWKEY_CG' when 20 then 'COL_ORIENTED_META' + when 21 then 'DDL_MERGE_CO' when 22 then 'DDL_MERGE_CG' when 23 then 'DDL_MEM_CO' + when 24 then 'DDL_MEM_CG' when 25 then 'DDL_MEM_MINI_SSTABLE' + when 26 then 'MDS_MINI' when 27 then 'MDS_MINOR' else 'INVALID' end) as TABLE_TYPE, M.LS_ID, @@ -61940,6 +61950,7 @@ def_table_schema( ) # 28232: GV$OB_QUERY_RESPONSE_TIME_HISTOGRAM # 28233: V$OB_QUERY_RESPONSE_TIME_HISTOGRAM +# 28234: DBA_OB_SPATIAL_COLUMNS # # 余留位置（此行之前占位） # 本区域占位建议：采用真实视图名进行占位 diff --git a/src/share/inner_table/sys_package/dbms_stats_body_mysql.sql b/src/share/inner_table/sys_package/dbms_stats_body_mysql.sql index 54c3292342..f975d69c4d 100644 --- a/src/share/inner_table/sys_package/dbms_stats_body_mysql.sql +++ b/src/share/inner_table/sys_package/dbms_stats_body_mysql.sql @@ -7,7 +7,7 @@ CREATE OR REPLACE PACKAGE BODY dbms_stats tabname VARCHAR(65535), partname VARCHAR(65535) DEFAULT NULL, estimate_percent DECIMAL DEFAULT AUTO_SAMPLE_SIZE, - block_sample BOOLEAN DEFAULT FALSE, + block_sample BOOLEAN DEFAULT NULL, method_opt VARCHAR(65535) DEFAULT DEFAULT_METHOD_OPT, degree DECIMAL DEFAULT NULL, granularity VARCHAR(65535) DEFAULT DEFAULT_GRANULARITY, @@ -24,7 +24,7 @@ CREATE OR REPLACE PACKAGE BODY dbms_stats PROCEDURE gather_schema_stats ( ownname VARCHAR(65535), estimate_percent DECIMAL DEFAULT AUTO_SAMPLE_SIZE, - block_sample BOOLEAN DEFAULT FALSE, + block_sample BOOLEAN DEFAULT NULL, method_opt VARCHAR(65535) DEFAULT DEFAULT_METHOD_OPT, degree DECIMAL DEFAULT NULL, granularity VARCHAR(65535) DEFAULT DEFAULT_GRANULARITY, diff --git a/src/share/inner_table/sys_package/dbms_stats_mysql.sql b/src/share/inner_table/sys_package/dbms_stats_mysql.sql index b4351fa0a4..d6f92f6d70 100644 --- a/src/share/inner_table/sys_package/dbms_stats_mysql.sql +++ b/src/share/inner_table/sys_package/dbms_stats_mysql.sql @@ -13,7 +13,7 @@ create or replace PACKAGE dbms_stats AUTHID CURRENT_USER tabname VARCHAR(65535), partname VARCHAR(65535) DEFAULT NULL, estimate_percent DECIMAL DEFAULT AUTO_SAMPLE_SIZE, - block_sample BOOLEAN DEFAULT FALSE, + block_sample BOOLEAN DEFAULT NULL, method_opt VARCHAR(65535) DEFAULT DEFAULT_METHOD_OPT, degree DECIMAL DEFAULT NULL, granularity VARCHAR(65535) DEFAULT DEFAULT_GRANULARITY, @@ -29,7 +29,7 @@ create or replace PACKAGE dbms_stats AUTHID CURRENT_USER PROCEDURE gather_schema_stats ( ownname VARCHAR(65535), estimate_percent DECIMAL DEFAULT AUTO_SAMPLE_SIZE, - block_sample BOOLEAN DEFAULT FALSE, + block_sample BOOLEAN DEFAULT NULL, method_opt VARCHAR(65535) DEFAULT DEFAULT_METHOD_OPT, degree DECIMAL DEFAULT NULL, granularity VARCHAR(65535) DEFAULT DEFAULT_GRANULARITY, diff --git a/src/share/io/ob_io_manager.cpp b/src/share/io/ob_io_manager.cpp index b24cc66d01..7c8dbd27c7 100644 --- a/src/share/io/ob_io_manager.cpp +++ b/src/share/io/ob_io_manager.cpp @@ -878,7 +878,7 @@ int ObTenantIOManager::inner_aio(const ObIOInfo &info, ObIOHandle &handle) } else if (OB_UNLIKELY(!is_working())) { ret = OB_STATE_NOT_MATCH; LOG_WARN("tenant not working", K(ret), K(tenant_id_)); - } else if (NULL != detector && detector->is_data_disk_has_fatal_error()) { + } else if (SLOG_IO != info.flag_.get_sys_module_id() && NULL != detector && detector->is_data_disk_has_fatal_error()) { ret = OB_DISK_HUNG; // for temporary positioning issue, get lbt of log replay LOG_DBA_ERROR(OB_DISK_HUNG, "msg", "disk has fatal error"); diff --git a/src/share/ob_cluster_version.h b/src/share/ob_cluster_version.h index 9df14df0de..22d96967a5 100644 --- a/src/share/ob_cluster_version.h +++ b/src/share/ob_cluster_version.h @@ -174,6 +174,7 @@ cal_version(const uint64_t major, const uint64_t minor, const uint64_t major_pat #define CLUSTER_VERSION_4_2_1_2 (oceanbase::common::cal_version(4, 2, 1, 2)) #define MOCK_CLUSTER_VERSION_4_2_1_3 (oceanbase::common::cal_version(4, 2, 1, 3)) #define MOCK_CLUSTER_VERSION_4_2_1_4 (oceanbase::common::cal_version(4, 2, 1, 4)) +#define MOCK_CLUSTER_VERSION_4_2_1_7 (oceanbase::common::cal_version(4, 2, 1, 7)) #define CLUSTER_VERSION_4_2_2_0 (oceanbase::common::cal_version(4, 2, 2, 0)) #define MOCK_CLUSTER_VERSION_4_2_2_1 (oceanbase::common::cal_version(4, 2, 2, 1)) #define MOCK_CLUSTER_VERSION_4_2_3_0 (oceanbase::common::cal_version(4, 2, 3, 0)) diff --git a/src/share/ob_ddl_common.h b/src/share/ob_ddl_common.h index 2d41fe29ba..f01b1cedda 100644 --- a/src/share/ob_ddl_common.h +++ b/src/share/ob_ddl_common.h @@ -108,6 +108,10 @@ enum ObDDLType DDL_CHANGE_COLUMN_NAME = 10003, DDL_DROP_COLUMN_INSTANT = 10004, DDL_ALTER_PARTITION_AUTO_SPLIT_ATTRIBUTE = 10005, // auto table auto partition // online + DDL_ADD_COLUMN_INSTANT = 10006, // add after/before column + DDL_MODIFY_COLUMN_ONLINE = 10007, + DDL_COMPOUND_ONLINE = 10008, + DDL_COMPOUND_INSTANT = 10009, ///< @note add new normal ddl type before this line DDL_MAX }; @@ -819,4 +823,3 @@ public: } // end namespace oceanbase #endif // OCEANBASE_SHARE_OB_DDL_COMMON_H - diff --git a/src/share/ob_debug_sync_point.h b/src/share/ob_debug_sync_point.h index d2997fda74..f5f0f04dd5 100755 --- a/src/share/ob_debug_sync_point.h +++ b/src/share/ob_debug_sync_point.h @@ -605,6 +605,7 @@ class ObString; ACT(BEFORE_START_TRANSFER_GET_TABLET_META,)\ ACT(BEFORE_ADD_REFRESH_SCHEMA_TASK,)\ ACT(BEFORE_ADD_ASYNC_REFRESH_SCHEMA_TASK,)\ + ACT(AFTER_MEMBERLIST_CHANGED,)\ ACT(MAX_DEBUG_SYNC_POINT,) DECLARE_ENUM(ObDebugSyncPoint, debug_sync_point, OB_DEBUG_SYNC_POINT_DEF); diff --git a/src/share/ob_dml_sql_splicer.cpp b/src/share/ob_dml_sql_splicer.cpp index ebb539b746..6d2f5370f4 100644 --- a/src/share/ob_dml_sql_splicer.cpp +++ b/src/share/ob_dml_sql_splicer.cpp @@ -1343,5 +1343,21 @@ int ObDMLExecHelper::check_row_exist(const char *table_name, return ret; } +int ObDMLSqlSplicer::add_long_double_column(const char *col_name, const double value) +{ + int ret = OB_SUCCESS; + const bool is_primary_key = false; + const bool is_null = false; + if (OB_ISNULL(col_name)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid column name", K(ret), KP(col_name)); + } else if (OB_FAIL(values_.append_fmt("%.17g", value))) { + LOG_WARN("append value failed", K(ret)); + } else if (OB_FAIL(add_column(is_primary_key, is_null, col_name))) { + LOG_WARN("add column failed", K(ret), K(is_primary_key), K(is_null), K(col_name)); + } + return ret; +} + } // end namespace share } // end namespace oceanbase diff --git a/src/share/ob_dml_sql_splicer.h b/src/share/ob_dml_sql_splicer.h index 5e890a6945..efa21e288b 100644 --- a/src/share/ob_dml_sql_splicer.h +++ b/src/share/ob_dml_sql_splicer.h @@ -100,6 +100,7 @@ public: int add_uint64_column(const char *col_name, const uint64_t value); int add_time_column(const char *col_name, const int64_t now, bool is_pk = false); int add_raw_time_column(const char *col_name, const int64_t now); + int add_long_double_column(const char *col_name, const double value); // mark end of one row int finish_row(); diff --git a/src/share/ob_errno.cpp b/src/share/ob_errno.cpp index 8033a2b476..e89e36e7b7 100644 --- a/src/share/ob_errno.cpp +++ b/src/share/ob_errno.cpp @@ -21969,6 +21969,18 @@ static const _error _error_OB_ERR_XML_PARENT_ALREADY_CONTAINS_CHILD = { .oracle_str_error = "ORA-31003: Parent %.*s already contains child entry %s%.*s", .oracle_str_user_error = "ORA-31003: Parent %.*s already contains child entry %s%.*s" }; +static const _error _error_OB_ERR_INVALID_VECTOR_DIM = { + .error_name = "OB_ERR_INVALID_VECTOR_DIM", + .error_cause = "Internal Error", + .error_solution = "Contact OceanBase Support", + .mysql_errno = -1, + .sqlstate = "22000", + .str_error = "Invalid dimension for vector.", + .str_user_error = "inconsistent dimension: expected %u got %u", + .oracle_errno = 932, + .oracle_str_error = "ORA-00932: inconsistent dimension", + .oracle_str_user_error = "ORA-00932: inconsistent dimension: expected %u got %u" +}; static const _error _error_OB_SERVER_IS_INIT = { .error_name = "OB_SERVER_IS_INIT", .error_cause = "Internal Error", @@ -29025,6 +29037,7 @@ struct ObStrErrorInit _errors[-OB_ERR_DUP_DEF_NAMESPACE] = &_error_OB_ERR_DUP_DEF_NAMESPACE; _errors[-OB_ERR_COMPARE_VARRAY_LOB_ATTR] = &_error_OB_ERR_COMPARE_VARRAY_LOB_ATTR; _errors[-OB_ERR_XML_PARENT_ALREADY_CONTAINS_CHILD] = &_error_OB_ERR_XML_PARENT_ALREADY_CONTAINS_CHILD; + _errors[-OB_ERR_INVALID_VECTOR_DIM] = &_error_OB_ERR_INVALID_VECTOR_DIM; _errors[-OB_SERVER_IS_INIT] = &_error_OB_SERVER_IS_INIT; _errors[-OB_SERVER_IS_STOPPING] = &_error_OB_SERVER_IS_STOPPING; _errors[-OB_PACKET_CHECKSUM_ERROR] = &_error_OB_PACKET_CHECKSUM_ERROR; @@ -29494,7 +29507,7 @@ namespace oceanbase { namespace common { -int g_all_ob_errnos[2263] = {0, -4000, -4001, -4002, -4003, -4004, -4005, -4006, -4007, -4008, -4009, -4010, -4011, -4012, -4013, -4014, -4015, -4016, -4017, -4018, -4019, -4020, -4021, -4022, -4023, -4024, -4025, -4026, -4027, -4028, -4029, -4030, -4031, -4032, -4033, -4034, -4035, -4036, -4037, -4038, -4039, -4041, -4042, -4043, -4044, -4045, -4046, -4047, -4048, -4049, -4050, -4051, -4052, -4053, -4054, -4055, -4057, -4058, -4060, -4061, -4062, -4063, -4064, -4065, -4066, -4067, -4068, -4070, -4071, -4072, -4073, -4074, -4075, -4076, -4077, -4078, -4080, -4081, -4084, -4085, -4090, -4097, -4098, -4099, -4100, -4101, -4102, -4103, -4104, -4105, -4106, -4107, -4108, -4109, -4110, -4111, -4112, -4113, -4114, -4115, -4116, -4117, -4118, -4119, -4120, -4121, -4122, -4123, -4124, -4125, -4126, -4127, -4128, -4133, -4138, -4139, -4142, -4143, -4144, -4146, -4147, -4149, -4150, -4151, -4152, -4153, -4154, -4155, -4156, -4157, -4158, -4159, -4160, -4161, -4162, -4163, -4164, -4165, -4166, -4167, -4168, -4169, -4170, -4171, -4172, -4173, -4174, -4175, -4176, -4177, -4178, -4179, -4180, -4181, -4182, -4183, -4184, -4185, -4186, -4187, -4188, -4189, -4190, -4191, -4192, -4200, -4201, -4204, -4205, -4206, -4207, -4208, -4209, -4210, -4211, -4212, -4213, -4214, -4215, -4216, -4217, -4218, -4219, -4220, -4221, -4222, -4223, -4224, -4225, -4226, -4227, -4228, -4229, -4230, -4231, -4232, -4233, -4234, -4235, -4236, -4237, -4238, -4239, -4240, -4241, -4242, -4243, -4244, -4245, -4246, -4247, -4248, -4249, -4250, -4251, -4252, -4253, -4254, -4255, -4256, -4257, -4258, -4260, -4261, -4262, -4263, -4264, -4265, -4266, -4267, -4268, -4269, -4270, -4271, -4273, -4274, -4275, -4276, -4277, -4278, -4279, -4280, -4281, -4282, -4283, -4284, -4285, -4286, -4287, -4288, -4289, -4290, -4291, -4292, -4293, -4294, -4295, -4296, -4297, -4298, -4299, -4300, -4301, -4302, -4303, -4304, -4305, -4306, -4307, -4308, -4309, -4310, -4311, -4312, -4313, -4314, -4315, -4316, -4317, -4318, -4319, -4320, -4321, -4322, -4323, -4324, -4325, -4326, -4327, -4328, -4329, -4330, -4331, -4332, -4333, -4334, -4335, -4336, -4337, -4338, -4339, -4340, -4341, -4342, -4343, -4344, -4345, -4346, -4347, -4348, -4349, -4350, -4351, -4352, -4353, -4354, -4355, -4356, -4357, -4358, -4359, -4360, -4361, -4362, -4363, -4364, -4365, -4366, -4367, -4368, -4369, -4370, -4371, -4372, -4373, -4374, -4375, -4376, -4377, -4378, -4379, -4380, -4381, -4382, -4383, -4385, -4386, -4387, -4388, -4389, -4390, -4391, -4392, -4393, -4394, -4395, -4396, -4397, -4398, -4399, -4400, -4401, -4402, -4403, -4505, -4507, -4510, -4512, -4515, -4517, -4518, -4519, -4523, -4524, -4525, -4526, -4527, -4528, -4529, -4530, -4531, -4532, -4533, -4537, -4538, -4539, -4540, -4541, -4542, -4543, -4544, -4545, -4546, -4547, -4548, -4549, -4550, -4551, -4552, -4553, -4554, -4600, -4601, -4602, -4603, -4604, -4605, -4606, -4607, -4608, -4609, -4610, -4611, -4613, -4614, -4615, -4620, -4621, -4622, -4623, -4624, -4625, -4626, -4628, -4629, -4630, -4631, -4632, -4633, -4634, -4636, -4637, -4638, -4639, -4640, -4641, -4642, -4643, -4644, -4645, -4646, -4647, -4648, -4649, -4650, -4651, -4652, -4653, -4654, -4655, -4656, -4657, -4658, -4659, -4660, -4661, -4662, -4663, -4664, -4665, -4666, -4667, -4668, -4669, -4670, -4671, -4672, -4673, -4674, -4675, -4676, -4677, -4678, -4679, -4680, -4681, -4682, -4683, -4684, -4685, -4686, -4687, -4688, -4689, -4690, -4691, -4692, -4693, -4694, -4695, -4696, -4697, -4698, -4699, -4700, -4701, -4702, -4703, -4704, -4705, -4706, -4707, -4708, -4709, -4710, -4711, -4712, -4713, -4714, -4715, -4716, -4717, -4718, -4719, -4720, -4721, -4722, -4723, -4724, -4725, -4726, -4727, -4728, -4729, -4730, -4731, -4732, -4733, -4734, -4735, -4736, -4737, -4738, -4739, -4740, -4741, -4742, -4743, -4744, -4745, -4746, -4747, -4748, -4749, -4750, -4751, -4752, -4753, -4754, -4755, -4756, -4757, -4758, -4759, -4760, -4761, -4762, -4763, -4764, -4765, -4766, -4767, -4768, -4769, -4770, -4771, -4772, -4773, -4774, -4775, -4776, -4777, -4778, -4779, -4780, -4781, -4782, -4783, -5000, -5001, -5002, -5003, -5006, -5007, -5008, -5010, -5011, -5012, -5014, -5015, -5016, -5017, -5018, -5019, -5020, -5022, -5023, -5024, -5025, -5026, -5027, -5028, -5029, -5030, -5031, -5032, -5034, -5035, -5036, -5037, -5038, -5039, -5040, -5041, -5042, -5043, -5044, -5046, -5047, -5050, -5051, -5052, -5053, -5054, -5055, -5056, -5057, -5058, -5059, -5061, -5063, -5064, -5065, -5066, -5067, -5068, -5069, -5070, -5071, -5072, -5073, -5074, -5080, -5081, -5083, -5084, -5085, -5086, -5087, -5088, -5089, -5090, -5091, -5092, -5093, -5094, -5095, -5096, -5097, -5098, -5099, -5100, -5101, -5102, -5103, -5104, -5105, -5106, -5107, -5108, -5109, -5110, -5111, -5112, -5113, -5114, -5115, -5116, -5117, -5118, -5119, -5120, -5121, -5122, -5123, -5124, -5125, -5130, -5131, -5133, -5134, -5135, -5136, -5137, -5138, -5139, -5140, -5142, -5143, -5144, -5145, -5146, -5147, -5148, -5149, -5150, -5151, -5153, -5154, -5155, -5156, -5157, -5158, -5159, -5160, -5161, -5162, -5163, -5164, -5165, -5166, -5167, -5168, -5169, -5170, -5171, -5172, -5173, -5174, -5175, -5176, -5177, -5178, -5179, -5180, -5181, -5182, -5183, -5184, -5185, -5187, -5188, -5189, -5190, -5191, -5192, -5193, -5194, -5195, -5196, -5197, -5198, -5199, -5200, -5201, -5202, -5203, -5204, -5205, -5206, -5207, -5208, -5209, -5210, -5211, -5212, -5213, -5214, -5215, -5216, -5217, -5218, -5219, -5220, -5221, -5222, -5223, -5224, -5225, -5226, -5227, -5228, -5229, -5230, -5231, -5233, -5234, -5235, -5236, -5237, -5238, -5239, -5240, -5241, -5242, -5243, -5244, -5245, -5246, -5247, -5248, -5249, -5250, -5251, -5252, -5253, -5254, -5255, -5256, -5257, -5258, -5259, -5260, -5261, -5262, -5263, -5264, -5265, -5266, -5267, -5268, -5269, -5270, -5271, -5272, -5273, -5274, -5275, -5276, -5277, -5278, -5279, -5280, -5281, -5282, -5283, -5284, -5285, -5286, -5287, -5288, -5289, -5290, -5291, -5292, -5293, -5294, -5295, -5296, -5297, -5298, -5299, -5300, -5301, -5302, -5303, -5304, -5305, -5306, -5307, -5308, -5309, -5310, -5311, -5312, -5313, -5314, -5315, -5316, -5317, -5318, -5319, -5320, -5321, -5322, -5323, -5324, -5325, -5326, -5327, -5328, -5329, -5330, -5331, -5332, -5333, -5334, -5335, -5336, -5337, -5338, -5339, -5340, -5341, -5342, -5343, -5344, -5345, -5346, -5347, -5348, -5349, -5350, -5351, -5352, -5353, -5354, -5355, -5356, -5357, -5358, -5359, -5360, -5361, -5362, -5363, -5364, -5365, -5366, -5367, -5368, -5369, -5370, -5371, -5372, -5373, -5374, -5375, -5376, -5377, -5378, -5379, -5380, -5381, -5382, -5383, -5384, -5385, -5386, -5387, -5388, -5389, -5390, -5400, -5401, -5402, -5403, -5404, -5405, -5406, -5407, -5408, -5409, -5410, -5411, -5412, -5413, -5414, -5415, -5416, -5417, -5418, -5419, -5420, -5421, -5422, -5423, -5424, -5425, -5426, -5427, -5428, -5429, -5430, -5431, -5432, -5433, -5434, -5435, -5436, -5437, -5438, -5439, -5440, -5441, -5442, -5443, -5444, -5445, -5446, -5447, -5448, -5449, -5450, -5451, -5452, -5453, -5454, -5455, -5456, -5457, -5458, -5459, -5460, -5461, -5462, -5463, -5464, -5465, -5466, -5467, -5468, -5469, -5470, -5471, -5472, -5473, -5474, -5475, -5476, -5477, -5478, -5479, -5480, -5481, -5482, -5483, -5484, -5485, -5486, -5487, -5488, -5489, -5490, -5491, -5492, -5493, -5494, -5495, -5496, -5497, -5498, -5499, -5500, -5501, -5502, -5503, -5504, -5505, -5506, -5507, -5508, -5509, -5510, -5511, -5512, -5513, -5514, -5515, -5516, -5517, -5518, -5519, -5520, -5521, -5522, -5540, -5541, -5542, -5543, -5544, -5545, -5546, -5547, -5548, -5549, -5550, -5551, -5552, -5553, -5554, -5555, -5556, -5557, -5558, -5559, -5560, -5561, -5562, -5563, -5564, -5565, -5566, -5567, -5568, -5569, -5570, -5571, -5572, -5573, -5574, -5575, -5576, -5577, -5578, -5579, -5580, -5581, -5582, -5583, -5584, -5585, -5586, -5587, -5588, -5589, -5590, -5591, -5592, -5593, -5594, -5595, -5596, -5597, -5598, -5599, -5600, -5601, -5602, -5603, -5604, -5605, -5607, -5608, -5609, -5610, -5611, -5612, -5613, -5614, -5615, -5616, -5617, -5618, -5619, -5620, -5621, -5622, -5623, -5624, -5625, -5626, -5627, -5628, -5629, -5630, -5631, -5632, -5633, -5634, -5635, -5636, -5637, -5638, -5639, -5640, -5641, -5642, -5643, -5644, -5645, -5646, -5647, -5648, -5649, -5650, -5651, -5652, -5653, -5654, -5655, -5656, -5657, -5658, -5659, -5660, -5661, -5662, -5663, -5664, -5665, -5666, -5667, -5668, -5671, -5672, -5673, -5674, -5675, -5676, -5677, -5678, -5679, -5680, -5681, -5682, -5683, -5684, -5685, -5686, -5687, -5688, -5689, -5690, -5691, -5692, -5693, -5694, -5695, -5696, -5697, -5698, -5699, -5700, -5701, -5702, -5703, -5704, -5705, -5706, -5707, -5708, -5709, -5710, -5711, -5712, -5713, -5714, -5715, -5716, -5717, -5718, -5719, -5720, -5721, -5722, -5723, -5724, -5725, -5726, -5727, -5728, -5729, -5730, -5731, -5732, -5733, -5734, -5735, -5736, -5737, -5738, -5739, -5740, -5741, -5742, -5743, -5744, -5745, -5746, -5747, -5748, -5749, -5750, -5751, -5752, -5753, -5754, -5755, -5756, -5757, -5758, -5759, -5760, -5761, -5762, -5763, -5764, -5765, -5766, -5768, -5769, -5770, -5771, -5772, -5773, -5774, -5777, -5778, -5779, -5780, -5781, -5785, -5786, -5787, -5788, -5789, -5790, -5791, -5792, -5793, -5794, -5795, -5796, -5797, -5798, -5799, -5800, -5801, -5802, -5803, -5804, -5805, -5806, -5807, -5808, -5809, -5810, -5811, -5812, -5813, -5814, -5815, -5816, -5817, -5818, -5819, -5820, -5821, -5822, -5823, -5824, -5825, -5826, -5827, -5828, -5829, -5830, -5831, -5832, -5833, -5834, -5835, -5836, -5837, -5838, -5839, -5840, -5841, -5842, -5843, -5844, -5845, -5846, -5847, -5848, -5849, -5850, -5851, -5852, -5853, -5854, -5855, -5856, -5857, -5858, -5859, -5860, -5861, -5862, -5863, -5864, -5865, -5866, -5867, -5868, -5869, -5870, -5871, -5872, -5873, -5874, -5875, -5876, -5877, -5878, -5879, -5880, -5881, -5882, -5883, -5884, -5885, -5886, -5887, -5888, -5889, -5890, -5891, -5892, -5893, -5894, -5895, -5896, -5897, -5898, -5899, -5900, -5901, -5902, -5903, -5904, -5905, -5906, -5907, -5908, -5909, -5910, -5911, -5912, -5913, -5914, -5915, -5916, -5917, -5918, -5919, -5920, -5921, -5922, -5923, -5924, -5925, -5926, -5927, -5928, -5929, -5930, -5931, -5932, -5933, -5934, -5935, -5936, -5937, -5938, -5939, -5940, -5941, -5942, -5943, -5944, -5945, -5946, -5947, -5948, -5949, -5950, -5951, -5952, -5953, -5954, -5955, -5956, -5957, -5958, -5959, -5960, -5961, -5962, -5963, -5964, -5965, -5966, -5967, -5968, -5969, -5970, -5971, -5972, -5973, -5974, -5975, -5976, -5977, -5978, -5979, -5980, -5981, -5982, -5983, -5984, -5985, -5986, -5987, -5988, -5989, -5990, -5991, -5992, -5993, -5994, -5995, -5996, -5997, -5998, -5999, -6000, -6001, -6002, -6003, -6004, -6005, -6006, -6201, -6202, -6203, -6204, -6205, -6206, -6207, -6208, -6209, -6210, -6211, -6212, -6213, -6214, -6215, -6219, -6220, -6221, -6222, -6223, -6224, -6225, -6226, -6227, -6228, -6229, -6230, -6231, -6232, -6233, -6234, -6235, -6236, -6237, -6238, -6239, -6240, -6241, -6242, -6243, -6244, -6245, -6246, -6247, -6248, -6249, -6250, -6251, -6252, -6253, -6254, -6255, -6256, -6257, -6258, -6259, -6260, -6261, -6262, -6263, -6264, -6265, -6266, -6267, -6268, -6269, -6270, -6271, -6272, -6273, -6274, -6275, -6276, -6277, -6278, -6279, -6280, -6281, -6282, -6283, -6284, -6285, -6301, -6302, -6303, -6304, -6305, -6306, -6307, -6308, -6309, -6310, -6311, -6312, -6313, -6314, -6315, -6316, -6317, -6318, -6319, -6320, -6321, -6322, -6323, -6324, -6325, -6326, -6327, -6328, -6329, -6330, -6331, -7000, -7001, -7002, -7003, -7004, -7005, -7006, -7007, -7010, -7011, -7012, -7013, -7014, -7015, -7021, -7022, -7024, -7025, -7026, -7027, -7029, -7030, -7031, -7032, -7033, -7034, -7035, -7036, -7037, -7038, -7039, -7040, -7041, -7100, -7101, -7102, -7103, -7104, -7105, -7106, -7107, -7108, -7109, -7110, -7111, -7112, -7113, -7114, -7115, -7116, -7117, -7118, -7119, -7120, -7121, -7122, -7123, -7124, -7201, -7202, -7203, -7204, -7205, -7206, -7207, -7208, -7209, -7210, -7211, -7212, -7213, -7214, -7215, -7216, -7217, -7218, -7219, -7220, -7221, -7222, -7223, -7224, -7225, -7226, -7227, -7228, -7229, -7230, -7231, -7232, -7233, -7234, -7235, -7236, -7237, -7238, -7239, -7240, -7241, -7242, -7243, -7244, -7246, -7247, -7248, -7249, -7250, -7251, -7252, -7253, -7254, -7255, -7256, -7257, -7258, -7259, -7260, -7261, -7262, -7263, -7264, -7265, -7266, -7267, -7268, -7269, -7270, -7271, -7272, -7273, -7274, -7275, -7276, -7277, -7278, -7279, -7280, -7281, -7282, -7283, -7284, -7285, -7286, -7287, -7288, -7289, -7290, -7291, -7292, -7293, -7294, -7295, -7296, -7297, -7298, -7299, -7300, -7301, -7302, -7402, -7403, -7404, -7405, -7406, -7407, -7408, -7409, -7410, -7411, -7412, -7413, -7414, -7415, -7416, -7417, -7418, -7419, -7420, -7421, -7422, -7423, -7424, -7425, -7426, -7427, -7428, -7429, -7430, -7431, -7432, -7433, -8001, -8002, -8003, -8004, -8005, -9001, -9002, -9003, -9004, -9005, -9006, -9007, -9008, -9009, -9010, -9011, -9012, -9013, -9014, -9015, -9016, -9017, -9018, -9019, -9020, -9022, -9023, -9024, -9025, -9026, -9027, -9028, -9029, -9030, -9031, -9032, -9033, -9034, -9035, -9036, -9037, -9038, -9039, -9040, -9041, -9042, -9043, -9044, -9045, -9046, -9047, -9048, -9049, -9050, -9051, -9052, -9053, -9054, -9057, -9058, -9059, -9060, -9061, -9062, -9063, -9064, -9065, -9066, -9069, -9070, -9071, -9072, -9073, -9074, -9075, -9076, -9077, -9078, -9079, -9080, -9081, -9082, -9083, -9084, -9085, -9086, -9087, -9088, -9089, -9090, -9091, -9092, -9093, -9094, -9095, -9096, -9097, -9098, -9099, -9100, -9101, -9102, -9103, -9104, -9105, -9106, -9107, -9108, -9109, -9110, -9111, -9112, -9113, -9114, -9115, -9116, -9117, -9118, -9119, -9120, -9121, -9122, -9123, -9200, -9201, -9202, -9203, -9501, -9502, -9503, -9504, -9505, -9506, -9507, -9508, -9509, -9510, -9512, -9513, -9514, -9515, -9516, -9518, -9519, -9520, -9521, -9522, -9523, -9524, -9525, -9526, -9527, -9528, -9529, -9530, -9531, -9532, -9533, -9534, -9535, -9536, -9537, -9538, -9539, -9540, -9541, -9542, -9543, -9544, -9545, -9546, -9547, -9548, -9549, -9550, -9551, -9552, -9553, -9554, -9555, -9556, -9557, -9558, -9559, -9560, -9561, -9562, -9563, -9564, -9565, -9566, -9567, -9568, -9569, -9570, -9571, -9572, -9573, -9574, -9575, -9576, -9577, -9578, -9579, -9580, -9581, -9582, -9583, -9584, -9585, -9586, -9587, -9588, -9589, -9590, -9591, -9592, -9593, -9594, -9595, -9596, -9597, -9598, -9599, -9600, -9601, -9602, -9603, -9604, -9605, -9606, -9607, -9608, -9609, -9610, -9611, -9612, -9613, -9614, -9615, -9616, -9617, -9618, -9619, -9620, -9621, -9622, -9623, -9624, -9625, -9626, -9627, -9628, -9629, -9630, -9631, -9632, -9633, -9634, -9635, -9636, -9637, -9638, -9639, -9640, -9641, -9642, -9643, -9644, -9645, -9646, -9647, -9648, -9649, -9650, -9651, -9652, -9653, -9654, -9655, -9656, -9657, -9658, -9659, -9660, -9661, -9662, -9663, -9664, -9665, -9666, -9667, -9668, -9669, -9670, -9671, -9672, -9673, -9674, -9675, -9676, -9677, -9678, -9679, -9680, -9681, -9682, -9683, -9684, -9685, -9686, -9687, -9688, -9689, -9690, -9691, -9692, -9693, -9694, -9695, -9696, -9697, -9698, -9699, -9700, -9701, -9702, -9703, -9704, -9705, -9706, -9707, -9708, -9709, -9710, -9711, -9712, -9713, -9714, -9715, -9716, -9717, -9718, -9719, -9720, -9721, -9722, -9723, -9724, -9725, -9726, -9727, -9728, -9729, -9730, -9731, -9732, -9733, -9734, -9735, -9736, -9737, -9738, -9739, -9740, -9741, -9742, -9743, -9744, -9745, -9746, -9747, -9748, -9749, -9750, -9751, -9752, -9753, -9754, -9755, -9756, -9757, -9758, -9759, -9760, -9761, -9762, -9763, -9764, -9765, -9766, -9767, -10500, -10501, -10502, -10503, -10504, -10505, -10506, -10507, -10508, -10509, -10510, -10511, -10512, -10513, -10514, -10515, -10516, -10650, -11000, -11001, -11002, -11003, -11004, -11005, -11006, -11007, -11008, -11009, -11010, -11011, -11012, -11013, -11014, -11015, -11016, -11017, -20000, -21000, -22998, -30926, -32491, -38104, -38105}; +int g_all_ob_errnos[2264] = {0, -4000, -4001, -4002, -4003, -4004, -4005, -4006, -4007, -4008, -4009, -4010, -4011, -4012, -4013, -4014, -4015, -4016, -4017, -4018, -4019, -4020, -4021, -4022, -4023, -4024, -4025, -4026, -4027, -4028, -4029, -4030, -4031, -4032, -4033, -4034, -4035, -4036, -4037, -4038, -4039, -4041, -4042, -4043, -4044, -4045, -4046, -4047, -4048, -4049, -4050, -4051, -4052, -4053, -4054, -4055, -4057, -4058, -4060, -4061, -4062, -4063, -4064, -4065, -4066, -4067, -4068, -4070, -4071, -4072, -4073, -4074, -4075, -4076, -4077, -4078, -4080, -4081, -4084, -4085, -4090, -4097, -4098, -4099, -4100, -4101, -4102, -4103, -4104, -4105, -4106, -4107, -4108, -4109, -4110, -4111, -4112, -4113, -4114, -4115, -4116, -4117, -4118, -4119, -4120, -4121, -4122, -4123, -4124, -4125, -4126, -4127, -4128, -4133, -4138, -4139, -4142, -4143, -4144, -4146, -4147, -4149, -4150, -4151, -4152, -4153, -4154, -4155, -4156, -4157, -4158, -4159, -4160, -4161, -4162, -4163, -4164, -4165, -4166, -4167, -4168, -4169, -4170, -4171, -4172, -4173, -4174, -4175, -4176, -4177, -4178, -4179, -4180, -4181, -4182, -4183, -4184, -4185, -4186, -4187, -4188, -4189, -4190, -4191, -4192, -4200, -4201, -4204, -4205, -4206, -4207, -4208, -4209, -4210, -4211, -4212, -4213, -4214, -4215, -4216, -4217, -4218, -4219, -4220, -4221, -4222, -4223, -4224, -4225, -4226, -4227, -4228, -4229, -4230, -4231, -4232, -4233, -4234, -4235, -4236, -4237, -4238, -4239, -4240, -4241, -4242, -4243, -4244, -4245, -4246, -4247, -4248, -4249, -4250, -4251, -4252, -4253, -4254, -4255, -4256, -4257, -4258, -4260, -4261, -4262, -4263, -4264, -4265, -4266, -4267, -4268, -4269, -4270, -4271, -4273, -4274, -4275, -4276, -4277, -4278, -4279, -4280, -4281, -4282, -4283, -4284, -4285, -4286, -4287, -4288, -4289, -4290, -4291, -4292, -4293, -4294, -4295, -4296, -4297, -4298, -4299, -4300, -4301, -4302, -4303, -4304, -4305, -4306, -4307, -4308, -4309, -4310, -4311, -4312, -4313, -4314, -4315, -4316, -4317, -4318, -4319, -4320, -4321, -4322, -4323, -4324, -4325, -4326, -4327, -4328, -4329, -4330, -4331, -4332, -4333, -4334, -4335, -4336, -4337, -4338, -4339, -4340, -4341, -4342, -4343, -4344, -4345, -4346, -4347, -4348, -4349, -4350, -4351, -4352, -4353, -4354, -4355, -4356, -4357, -4358, -4359, -4360, -4361, -4362, -4363, -4364, -4365, -4366, -4367, -4368, -4369, -4370, -4371, -4372, -4373, -4374, -4375, -4376, -4377, -4378, -4379, -4380, -4381, -4382, -4383, -4385, -4386, -4387, -4388, -4389, -4390, -4391, -4392, -4393, -4394, -4395, -4396, -4397, -4398, -4399, -4400, -4401, -4402, -4403, -4505, -4507, -4510, -4512, -4515, -4517, -4518, -4519, -4523, -4524, -4525, -4526, -4527, -4528, -4529, -4530, -4531, -4532, -4533, -4537, -4538, -4539, -4540, -4541, -4542, -4543, -4544, -4545, -4546, -4547, -4548, -4549, -4550, -4551, -4552, -4553, -4554, -4600, -4601, -4602, -4603, -4604, -4605, -4606, -4607, -4608, -4609, -4610, -4611, -4613, -4614, -4615, -4620, -4621, -4622, -4623, -4624, -4625, -4626, -4628, -4629, -4630, -4631, -4632, -4633, -4634, -4636, -4637, -4638, -4639, -4640, -4641, -4642, -4643, -4644, -4645, -4646, -4647, -4648, -4649, -4650, -4651, -4652, -4653, -4654, -4655, -4656, -4657, -4658, -4659, -4660, -4661, -4662, -4663, -4664, -4665, -4666, -4667, -4668, -4669, -4670, -4671, -4672, -4673, -4674, -4675, -4676, -4677, -4678, -4679, -4680, -4681, -4682, -4683, -4684, -4685, -4686, -4687, -4688, -4689, -4690, -4691, -4692, -4693, -4694, -4695, -4696, -4697, -4698, -4699, -4700, -4701, -4702, -4703, -4704, -4705, -4706, -4707, -4708, -4709, -4710, -4711, -4712, -4713, -4714, -4715, -4716, -4717, -4718, -4719, -4720, -4721, -4722, -4723, -4724, -4725, -4726, -4727, -4728, -4729, -4730, -4731, -4732, -4733, -4734, -4735, -4736, -4737, -4738, -4739, -4740, -4741, -4742, -4743, -4744, -4745, -4746, -4747, -4748, -4749, -4750, -4751, -4752, -4753, -4754, -4755, -4756, -4757, -4758, -4759, -4760, -4761, -4762, -4763, -4764, -4765, -4766, -4767, -4768, -4769, -4770, -4771, -4772, -4773, -4774, -4775, -4776, -4777, -4778, -4779, -4780, -4781, -4782, -4783, -5000, -5001, -5002, -5003, -5006, -5007, -5008, -5010, -5011, -5012, -5014, -5015, -5016, -5017, -5018, -5019, -5020, -5022, -5023, -5024, -5025, -5026, -5027, -5028, -5029, -5030, -5031, -5032, -5034, -5035, -5036, -5037, -5038, -5039, -5040, -5041, -5042, -5043, -5044, -5046, -5047, -5050, -5051, -5052, -5053, -5054, -5055, -5056, -5057, -5058, -5059, -5061, -5063, -5064, -5065, -5066, -5067, -5068, -5069, -5070, -5071, -5072, -5073, -5074, -5080, -5081, -5083, -5084, -5085, -5086, -5087, -5088, -5089, -5090, -5091, -5092, -5093, -5094, -5095, -5096, -5097, -5098, -5099, -5100, -5101, -5102, -5103, -5104, -5105, -5106, -5107, -5108, -5109, -5110, -5111, -5112, -5113, -5114, -5115, -5116, -5117, -5118, -5119, -5120, -5121, -5122, -5123, -5124, -5125, -5130, -5131, -5133, -5134, -5135, -5136, -5137, -5138, -5139, -5140, -5142, -5143, -5144, -5145, -5146, -5147, -5148, -5149, -5150, -5151, -5153, -5154, -5155, -5156, -5157, -5158, -5159, -5160, -5161, -5162, -5163, -5164, -5165, -5166, -5167, -5168, -5169, -5170, -5171, -5172, -5173, -5174, -5175, -5176, -5177, -5178, -5179, -5180, -5181, -5182, -5183, -5184, -5185, -5187, -5188, -5189, -5190, -5191, -5192, -5193, -5194, -5195, -5196, -5197, -5198, -5199, -5200, -5201, -5202, -5203, -5204, -5205, -5206, -5207, -5208, -5209, -5210, -5211, -5212, -5213, -5214, -5215, -5216, -5217, -5218, -5219, -5220, -5221, -5222, -5223, -5224, -5225, -5226, -5227, -5228, -5229, -5230, -5231, -5233, -5234, -5235, -5236, -5237, -5238, -5239, -5240, -5241, -5242, -5243, -5244, -5245, -5246, -5247, -5248, -5249, -5250, -5251, -5252, -5253, -5254, -5255, -5256, -5257, -5258, -5259, -5260, -5261, -5262, -5263, -5264, -5265, -5266, -5267, -5268, -5269, -5270, -5271, -5272, -5273, -5274, -5275, -5276, -5277, -5278, -5279, -5280, -5281, -5282, -5283, -5284, -5285, -5286, -5287, -5288, -5289, -5290, -5291, -5292, -5293, -5294, -5295, -5296, -5297, -5298, -5299, -5300, -5301, -5302, -5303, -5304, -5305, -5306, -5307, -5308, -5309, -5310, -5311, -5312, -5313, -5314, -5315, -5316, -5317, -5318, -5319, -5320, -5321, -5322, -5323, -5324, -5325, -5326, -5327, -5328, -5329, -5330, -5331, -5332, -5333, -5334, -5335, -5336, -5337, -5338, -5339, -5340, -5341, -5342, -5343, -5344, -5345, -5346, -5347, -5348, -5349, -5350, -5351, -5352, -5353, -5354, -5355, -5356, -5357, -5358, -5359, -5360, -5361, -5362, -5363, -5364, -5365, -5366, -5367, -5368, -5369, -5370, -5371, -5372, -5373, -5374, -5375, -5376, -5377, -5378, -5379, -5380, -5381, -5382, -5383, -5384, -5385, -5386, -5387, -5388, -5389, -5390, -5400, -5401, -5402, -5403, -5404, -5405, -5406, -5407, -5408, -5409, -5410, -5411, -5412, -5413, -5414, -5415, -5416, -5417, -5418, -5419, -5420, -5421, -5422, -5423, -5424, -5425, -5426, -5427, -5428, -5429, -5430, -5431, -5432, -5433, -5434, -5435, -5436, -5437, -5438, -5439, -5440, -5441, -5442, -5443, -5444, -5445, -5446, -5447, -5448, -5449, -5450, -5451, -5452, -5453, -5454, -5455, -5456, -5457, -5458, -5459, -5460, -5461, -5462, -5463, -5464, -5465, -5466, -5467, -5468, -5469, -5470, -5471, -5472, -5473, -5474, -5475, -5476, -5477, -5478, -5479, -5480, -5481, -5482, -5483, -5484, -5485, -5486, -5487, -5488, -5489, -5490, -5491, -5492, -5493, -5494, -5495, -5496, -5497, -5498, -5499, -5500, -5501, -5502, -5503, -5504, -5505, -5506, -5507, -5508, -5509, -5510, -5511, -5512, -5513, -5514, -5515, -5516, -5517, -5518, -5519, -5520, -5521, -5522, -5540, -5541, -5542, -5543, -5544, -5545, -5546, -5547, -5548, -5549, -5550, -5551, -5552, -5553, -5554, -5555, -5556, -5557, -5558, -5559, -5560, -5561, -5562, -5563, -5564, -5565, -5566, -5567, -5568, -5569, -5570, -5571, -5572, -5573, -5574, -5575, -5576, -5577, -5578, -5579, -5580, -5581, -5582, -5583, -5584, -5585, -5586, -5587, -5588, -5589, -5590, -5591, -5592, -5593, -5594, -5595, -5596, -5597, -5598, -5599, -5600, -5601, -5602, -5603, -5604, -5605, -5607, -5608, -5609, -5610, -5611, -5612, -5613, -5614, -5615, -5616, -5617, -5618, -5619, -5620, -5621, -5622, -5623, -5624, -5625, -5626, -5627, -5628, -5629, -5630, -5631, -5632, -5633, -5634, -5635, -5636, -5637, -5638, -5639, -5640, -5641, -5642, -5643, -5644, -5645, -5646, -5647, -5648, -5649, -5650, -5651, -5652, -5653, -5654, -5655, -5656, -5657, -5658, -5659, -5660, -5661, -5662, -5663, -5664, -5665, -5666, -5667, -5668, -5671, -5672, -5673, -5674, -5675, -5676, -5677, -5678, -5679, -5680, -5681, -5682, -5683, -5684, -5685, -5686, -5687, -5688, -5689, -5690, -5691, -5692, -5693, -5694, -5695, -5696, -5697, -5698, -5699, -5700, -5701, -5702, -5703, -5704, -5705, -5706, -5707, -5708, -5709, -5710, -5711, -5712, -5713, -5714, -5715, -5716, -5717, -5718, -5719, -5720, -5721, -5722, -5723, -5724, -5725, -5726, -5727, -5728, -5729, -5730, -5731, -5732, -5733, -5734, -5735, -5736, -5737, -5738, -5739, -5740, -5741, -5742, -5743, -5744, -5745, -5746, -5747, -5748, -5749, -5750, -5751, -5752, -5753, -5754, -5755, -5756, -5757, -5758, -5759, -5760, -5761, -5762, -5763, -5764, -5765, -5766, -5768, -5769, -5770, -5771, -5772, -5773, -5774, -5777, -5778, -5779, -5780, -5781, -5785, -5786, -5787, -5788, -5789, -5790, -5791, -5792, -5793, -5794, -5795, -5796, -5797, -5798, -5799, -5800, -5801, -5802, -5803, -5804, -5805, -5806, -5807, -5808, -5809, -5810, -5811, -5812, -5813, -5814, -5815, -5816, -5817, -5818, -5819, -5820, -5821, -5822, -5823, -5824, -5825, -5826, -5827, -5828, -5829, -5830, -5831, -5832, -5833, -5834, -5835, -5836, -5837, -5838, -5839, -5840, -5841, -5842, -5843, -5844, -5845, -5846, -5847, -5848, -5849, -5850, -5851, -5852, -5853, -5854, -5855, -5856, -5857, -5858, -5859, -5860, -5861, -5862, -5863, -5864, -5865, -5866, -5867, -5868, -5869, -5870, -5871, -5872, -5873, -5874, -5875, -5876, -5877, -5878, -5879, -5880, -5881, -5882, -5883, -5884, -5885, -5886, -5887, -5888, -5889, -5890, -5891, -5892, -5893, -5894, -5895, -5896, -5897, -5898, -5899, -5900, -5901, -5902, -5903, -5904, -5905, -5906, -5907, -5908, -5909, -5910, -5911, -5912, -5913, -5914, -5915, -5916, -5917, -5918, -5919, -5920, -5921, -5922, -5923, -5924, -5925, -5926, -5927, -5928, -5929, -5930, -5931, -5932, -5933, -5934, -5935, -5936, -5937, -5938, -5939, -5940, -5941, -5942, -5943, -5944, -5945, -5946, -5947, -5948, -5949, -5950, -5951, -5952, -5953, -5954, -5955, -5956, -5957, -5958, -5959, -5960, -5961, -5962, -5963, -5964, -5965, -5966, -5967, -5968, -5969, -5970, -5971, -5972, -5973, -5974, -5975, -5976, -5977, -5978, -5979, -5980, -5981, -5982, -5983, -5984, -5985, -5986, -5987, -5988, -5989, -5990, -5991, -5992, -5993, -5994, -5995, -5996, -5997, -5998, -5999, -6000, -6001, -6002, -6003, -6004, -6005, -6006, -6201, -6202, -6203, -6204, -6205, -6206, -6207, -6208, -6209, -6210, -6211, -6212, -6213, -6214, -6215, -6219, -6220, -6221, -6222, -6223, -6224, -6225, -6226, -6227, -6228, -6229, -6230, -6231, -6232, -6233, -6234, -6235, -6236, -6237, -6238, -6239, -6240, -6241, -6242, -6243, -6244, -6245, -6246, -6247, -6248, -6249, -6250, -6251, -6252, -6253, -6254, -6255, -6256, -6257, -6258, -6259, -6260, -6261, -6262, -6263, -6264, -6265, -6266, -6267, -6268, -6269, -6270, -6271, -6272, -6273, -6274, -6275, -6276, -6277, -6278, -6279, -6280, -6281, -6282, -6283, -6284, -6285, -6301, -6302, -6303, -6304, -6305, -6306, -6307, -6308, -6309, -6310, -6311, -6312, -6313, -6314, -6315, -6316, -6317, -6318, -6319, -6320, -6321, -6322, -6323, -6324, -6325, -6326, -6327, -6328, -6329, -6330, -6331, -7000, -7001, -7002, -7003, -7004, -7005, -7006, -7007, -7010, -7011, -7012, -7013, -7014, -7015, -7021, -7022, -7024, -7025, -7026, -7027, -7029, -7030, -7031, -7032, -7033, -7034, -7035, -7036, -7037, -7038, -7039, -7040, -7041, -7100, -7101, -7102, -7103, -7104, -7105, -7106, -7107, -7108, -7109, -7110, -7111, -7112, -7113, -7114, -7115, -7116, -7117, -7118, -7119, -7120, -7121, -7122, -7123, -7124, -7201, -7202, -7203, -7204, -7205, -7206, -7207, -7208, -7209, -7210, -7211, -7212, -7213, -7214, -7215, -7216, -7217, -7218, -7219, -7220, -7221, -7222, -7223, -7224, -7225, -7226, -7227, -7228, -7229, -7230, -7231, -7232, -7233, -7234, -7235, -7236, -7237, -7238, -7239, -7240, -7241, -7242, -7243, -7244, -7246, -7247, -7248, -7249, -7250, -7251, -7252, -7253, -7254, -7255, -7256, -7257, -7258, -7259, -7260, -7261, -7262, -7263, -7264, -7265, -7266, -7267, -7268, -7269, -7270, -7271, -7272, -7273, -7274, -7275, -7276, -7277, -7278, -7279, -7280, -7281, -7282, -7283, -7284, -7285, -7286, -7287, -7288, -7289, -7290, -7291, -7292, -7293, -7294, -7295, -7296, -7297, -7298, -7299, -7300, -7301, -7302, -7402, -7403, -7404, -7405, -7406, -7407, -7408, -7409, -7410, -7411, -7412, -7413, -7414, -7415, -7416, -7417, -7418, -7419, -7420, -7421, -7422, -7423, -7424, -7425, -7426, -7427, -7428, -7429, -7430, -7431, -7432, -7433, -7600, -8001, -8002, -8003, -8004, -8005, -9001, -9002, -9003, -9004, -9005, -9006, -9007, -9008, -9009, -9010, -9011, -9012, -9013, -9014, -9015, -9016, -9017, -9018, -9019, -9020, -9022, -9023, -9024, -9025, -9026, -9027, -9028, -9029, -9030, -9031, -9032, -9033, -9034, -9035, -9036, -9037, -9038, -9039, -9040, -9041, -9042, -9043, -9044, -9045, -9046, -9047, -9048, -9049, -9050, -9051, -9052, -9053, -9054, -9057, -9058, -9059, -9060, -9061, -9062, -9063, -9064, -9065, -9066, -9069, -9070, -9071, -9072, -9073, -9074, -9075, -9076, -9077, -9078, -9079, -9080, -9081, -9082, -9083, -9084, -9085, -9086, -9087, -9088, -9089, -9090, -9091, -9092, -9093, -9094, -9095, -9096, -9097, -9098, -9099, -9100, -9101, -9102, -9103, -9104, -9105, -9106, -9107, -9108, -9109, -9110, -9111, -9112, -9113, -9114, -9115, -9116, -9117, -9118, -9119, -9120, -9121, -9122, -9123, -9200, -9201, -9202, -9203, -9501, -9502, -9503, -9504, -9505, -9506, -9507, -9508, -9509, -9510, -9512, -9513, -9514, -9515, -9516, -9518, -9519, -9520, -9521, -9522, -9523, -9524, -9525, -9526, -9527, -9528, -9529, -9530, -9531, -9532, -9533, -9534, -9535, -9536, -9537, -9538, -9539, -9540, -9541, -9542, -9543, -9544, -9545, -9546, -9547, -9548, -9549, -9550, -9551, -9552, -9553, -9554, -9555, -9556, -9557, -9558, -9559, -9560, -9561, -9562, -9563, -9564, -9565, -9566, -9567, -9568, -9569, -9570, -9571, -9572, -9573, -9574, -9575, -9576, -9577, -9578, -9579, -9580, -9581, -9582, -9583, -9584, -9585, -9586, -9587, -9588, -9589, -9590, -9591, -9592, -9593, -9594, -9595, -9596, -9597, -9598, -9599, -9600, -9601, -9602, -9603, -9604, -9605, -9606, -9607, -9608, -9609, -9610, -9611, -9612, -9613, -9614, -9615, -9616, -9617, -9618, -9619, -9620, -9621, -9622, -9623, -9624, -9625, -9626, -9627, -9628, -9629, -9630, -9631, -9632, -9633, -9634, -9635, -9636, -9637, -9638, -9639, -9640, -9641, -9642, -9643, -9644, -9645, -9646, -9647, -9648, -9649, -9650, -9651, -9652, -9653, -9654, -9655, -9656, -9657, -9658, -9659, -9660, -9661, -9662, -9663, -9664, -9665, -9666, -9667, -9668, -9669, -9670, -9671, -9672, -9673, -9674, -9675, -9676, -9677, -9678, -9679, -9680, -9681, -9682, -9683, -9684, -9685, -9686, -9687, -9688, -9689, -9690, -9691, -9692, -9693, -9694, -9695, -9696, -9697, -9698, -9699, -9700, -9701, -9702, -9703, -9704, -9705, -9706, -9707, -9708, -9709, -9710, -9711, -9712, -9713, -9714, -9715, -9716, -9717, -9718, -9719, -9720, -9721, -9722, -9723, -9724, -9725, -9726, -9727, -9728, -9729, -9730, -9731, -9732, -9733, -9734, -9735, -9736, -9737, -9738, -9739, -9740, -9741, -9742, -9743, -9744, -9745, -9746, -9747, -9748, -9749, -9750, -9751, -9752, -9753, -9754, -9755, -9756, -9757, -9758, -9759, -9760, -9761, -9762, -9763, -9764, -9765, -9766, -9767, -10500, -10501, -10502, -10503, -10504, -10505, -10506, -10507, -10508, -10509, -10510, -10511, -10512, -10513, -10514, -10515, -10516, -10650, -11000, -11001, -11002, -11003, -11004, -11005, -11006, -11007, -11008, -11009, -11010, -11011, -11012, -11013, -11014, -11015, -11016, -11017, -20000, -21000, -22998, -30926, -32491, -38104, -38105}; const char *ob_error_name(const int err) { const char *ret = "Unknown error"; diff --git a/src/share/ob_errno.def b/src/share/ob_errno.def index c7591eb033..85e45b4115 100755 --- a/src/share/ob_errno.def +++ b/src/share/ob_errno.def @@ -9,6 +9,7 @@ // 负载均衡(Transfer)错误码值域[-7100, -7200) // GIS错误码值域 [-7201, -7400) // XML错误码值域 [-7400, -7600) +// ARRAY错误码值域 [-7600, -7700) // 致命错误[-8000, -9000)，客户端收到8xxx错误，需要关闭SQL连接 // Storage错误码值域 [-9000, -9500) // PL/SQL错误码值域 [-9500, -10000) @@ -2136,6 +2137,26 @@ DEFINE_ORACLE_ERROR(OB_ERR_XML_PARENT_ALREADY_CONTAINS_CHILD, -7433, -1, "42000" // 4. 本文件修改完成后，需要调用gen_errno.pl，生成ob_errno.h文件 //////////////////////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ARRAY错误码值域 [-7600, -7700) +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +DEFINE_ORACLE_ERROR_EXT_DEP(OB_ERR_INVALID_VECTOR_DIM, -7600, -1, "22000", "Invalid dimension for vector.", "inconsistent dimension: expected %u got %u", 932, "inconsistent dimension", "inconsistent dimension: expected %u got %u"); + +// 余留位置 +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ARRAY错误码值域 [-7600, -7700) +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////// 新增(占位)须知 //////////////////////////////////// +// 1. 新增错误码需要先在master分支提交，保证master是所有版本的超集，避免出现错误码冲突 +// 2. 不支持以注释方式“占位”错误码，如果要新增错误码，直接提交完整的错误码定义即可 +// 3. 不同的错误码区间有不同含义，请在合理区间内顺序定义错误码。区间定义见文件头。 +// 每个区间内，请在“余留位置”行之前定义新错误码 +// 如果没有合适区间，请联系 @修铭 定义一个新的区间 +// 4. 本文件修改完成后，需要调用gen_errno.pl，生成ob_errno.h文件 +//////////////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////// // 致命错误[-8000, -9000)，客户端收到8xxx错误，需要关闭SQL连接 //////////////////////////////////////////////////////////////// diff --git a/src/share/ob_errno.h b/src/share/ob_errno.h index d0939ef6ca..9e9cfb200a 100644 --- a/src/share/ob_errno.h +++ b/src/share/ob_errno.h @@ -3720,6 +3720,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_DUP_DEF_NAMESPACE__USER_ERROR_MSG "XQST0066 - duplicate default namespace definition - %s." #define OB_ERR_COMPARE_VARRAY_LOB_ATTR__USER_ERROR_MSG "cannot compare VARRAY or LOB attributes of an object type" #define OB_ERR_XML_PARENT_ALREADY_CONTAINS_CHILD__USER_ERROR_MSG "Parent %.*s already contains child entry %s%.*s" +#define OB_ERR_INVALID_VECTOR_DIM__USER_ERROR_MSG "inconsistent dimension: expected %u got %u" #define OB_SERVER_IS_INIT__USER_ERROR_MSG "Server is initializing" #define OB_SERVER_IS_STOPPING__USER_ERROR_MSG "Server is stopping" #define OB_PACKET_CHECKSUM_ERROR__USER_ERROR_MSG "Packet checksum error" @@ -5987,6 +5988,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_DUP_DEF_NAMESPACE__ORA_USER_ERROR_MSG "ORA-19118: XQST0066 - duplicate default namespace definition - %s." #define OB_ERR_COMPARE_VARRAY_LOB_ATTR__ORA_USER_ERROR_MSG "ORA-22901: cannot compare VARRAY or LOB attributes of an object type" #define OB_ERR_XML_PARENT_ALREADY_CONTAINS_CHILD__ORA_USER_ERROR_MSG "ORA-31003: Parent %.*s already contains child entry %s%.*s" +#define OB_ERR_INVALID_VECTOR_DIM__ORA_USER_ERROR_MSG "ORA-00932: inconsistent dimension: expected %u got %u" #define OB_SERVER_IS_INIT__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -8001, Server is initializing" #define OB_SERVER_IS_STOPPING__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -8002, Server is stopping" #define OB_PACKET_CHECKSUM_ERROR__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -8003, Packet checksum error" @@ -6425,7 +6427,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_DATA_TOO_LONG_MSG_FMT_V2__ORA_USER_ERROR_MSG "ORA-12899: value too large for column %.*s (actual: %ld, maximum: %ld)" #define OB_ERR_INVALID_DATE_MSG_FMT_V2__ORA_USER_ERROR_MSG "ORA-01861: Incorrect datetime value for column '%.*s' at row %ld" -extern int g_all_ob_errnos[2263]; +extern int g_all_ob_errnos[2264]; const char *ob_error_name(const int oberr); const char* ob_error_cause(const int oberr); diff --git a/src/share/ob_fts_index_builder_util.cpp b/src/share/ob_fts_index_builder_util.cpp index 330aeb84a8..bb184b81f8 100644 --- a/src/share/ob_fts_index_builder_util.cpp +++ b/src/share/ob_fts_index_builder_util.cpp @@ -1511,6 +1511,20 @@ int ObFtsIndexBuilderUtil::get_doc_id_col( return ret; } +int ObFtsIndexBuilderUtil::check_fts_or_multivalue_index_allowed( + ObTableSchema &data_schema) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!data_schema.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(data_schema)); + } else if (data_schema.is_partitioned_table() && data_schema.is_heap_table()) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "create full-text or multi-value index on partition table without primary key"); + } + return ret; +} + int ObFtsIndexBuilderUtil::get_word_segment_col( const ObTableSchema &data_schema, const obrpc::ObCreateIndexArg *index_arg, diff --git a/src/share/ob_fts_index_builder_util.h b/src/share/ob_fts_index_builder_util.h index cef2252c1c..1eca7c4627 100644 --- a/src/share/ob_fts_index_builder_util.h +++ b/src/share/ob_fts_index_builder_util.h @@ -68,6 +68,8 @@ public: static int get_doc_id_col( const ObTableSchema &data_schema, const ObColumnSchemaV2 *&doc_id_col); + static int check_fts_or_multivalue_index_allowed( + ObTableSchema &data_schema); private: static int check_ft_cols( const obrpc::ObCreateIndexArg *index_arg, diff --git a/src/share/ob_index_builder_util.cpp b/src/share/ob_index_builder_util.cpp index 05ca32a881..9a382cab9f 100644 --- a/src/share/ob_index_builder_util.cpp +++ b/src/share/ob_index_builder_util.cpp @@ -660,11 +660,15 @@ int ObIndexBuilderUtil::adjust_expr_index_args( LOG_WARN("push back mbr column to gen columns failed", K(ret)); } } else if (is_fts_index(arg.index_type_)) { - if (OB_FAIL(ObFtsIndexBuilderUtil::adjust_fts_args(arg, data_schema, gen_columns))) { + if (OB_FAIL(ObFtsIndexBuilderUtil::check_fts_or_multivalue_index_allowed(data_schema))) { + LOG_WARN("fail to check fts index allowed", K(ret)); + } else if (OB_FAIL(ObFtsIndexBuilderUtil::adjust_fts_args(arg, data_schema, gen_columns))) { LOG_WARN("failed to adjust fts args", K(ret)); } } else if (is_multivalue_index(arg.index_type_)) { - if (OB_FAIL(ObMulValueIndexBuilderUtil::adjust_mulvalue_index_args(arg, data_schema, gen_columns))) { + if (OB_FAIL(ObFtsIndexBuilderUtil::check_fts_or_multivalue_index_allowed(data_schema))) { + LOG_WARN("fail to check multivalue index allowed", K(ret)); + } else if (OB_FAIL(ObMulValueIndexBuilderUtil::adjust_mulvalue_index_args(arg, data_schema, gen_columns))) { LOG_WARN("failed to adjust multivalue args", K(ret)); } } else if (OB_FAIL(adjust_ordinary_index_column_args(arg, data_schema, allocator, gen_columns))) { diff --git a/src/share/parameter/default_parameter.json b/src/share/parameter/default_parameter.json index da41ad10c8..a6d586d402 100644 --- a/src/share/parameter/default_parameter.json +++ b/src/share/parameter/default_parameter.json @@ -4,13 +4,32 @@ "comment" : "for workloads like trade, payment core system, internet high throughput application, etc. no restrictions like foreign key, no stored procedure, no long transaction, no large transaction, no complex join, no complex subquery", "parameters": { "cluster": [ - + { + "name": "_enable_defensive_check", + "value": 0, + "comment": "disabling the defensive check feature in production environment can result in a 10% performance improvement for DML operations" + }, + { + "name": "enable_syslog_recycle", + "value": 1, + "comment": "enable syslog auto recycle can prevent log files from filling up disk space" + }, + { + "name": "max_syslog_file_count", + "value": 300, + "comment": "when enable_syslog_recycle is enabled, should set this value to a proper value. 300 is an empirical value." + } ], "tenant": [ { - "name":"_rowsets_max_rows", + "name": "_rowsets_max_rows", "value": 1, - "comment":"for simple OLTP workloads, rowset = 1 is most effective." + "comment": "for simple OLTP workloads, rowset = 1 is most effective." + }, + { + "name": "log_transport_compress_all", + "value": 1, + "comment": "In scenarios with limited bandwidth, network bandwidth can be saved with a small amount of CPU overhead through RPC compression" } ] } @@ -21,16 +40,36 @@ "parameters": { "cluster": [ { - "name":"large_query_threshold", - "value":"600s", - "comment":"for complex OLTP scenario, some query will run for very long time." + "name": "large_query_threshold", + "value": "600s", + "comment": "for complex OLTP scenario, some query will run for very long time." + }, + { + "name": "_enable_defensive_check", + "value": 0, + "comment": "disabling the defensive check feature in production environment can result in a 3% performance improvement" + }, + { + "name": "enable_syslog_recycle", + "value": 1, + "comment": "enable syslog auto recycle can prevent log files from filling up disk space" + }, + { + "name": "max_syslog_file_count", + "value": 300, + "comment": "when enable_syslog_recycle is enabled, should set this value to a proper value. 300 is an empirical value." } ], "tenant": [ { - "name":"_rowsets_max_rows", + "name": "_rowsets_max_rows", "value": 4, - "comment":"for complex OLTP workloads, rowset = 4 is most effective." + "comment": "for complex OLTP workloads, rowset = 4 is most effective." + }, + { + "name": "log_transport_compress_all", + "value": 1, + "comment": "In scenarios with limited bandwidth, network bandwidth can be saved with a small amount of CPU overhead through RPC compression" } ] } @@ -46,26 +85,26 @@ "comment": "disable trace log for better AP performance" }, { - "name":"trace_log_slow_query_watermark", - "value":"7d", - "comment":"7 days. no 'slow query' concept for AP query" + "name": "trace_log_slow_query_watermark", + "value": "7d", + "comment": "7 days. no 'slow query' concept for AP query" }, { - "name":"large_query_threshold", - "value":"0ms", - "comment":"disable large query detection for AP query" + "name": "large_query_threshold", + "value": "0ms", + "comment": "disable large query detection for AP query" } ], "tenant": [ { - "name":"default_table_store_format", - "value":"column", - "comment":"default to column format for AP" + "name": "default_table_store_format", + "value": "column", + "comment": "default to column format for AP" }, { - "name":"_rowsets_max_rows", + "name": "_rowsets_max_rows", "value": 256, - "comment":"for classic OLAP workloads, rowset 256 is adequate" + "comment": "for classic OLAP workloads, rowset 256 is adequate" } ] } @@ -76,9 +115,9 @@ "parameters": { "cluster": [ { - "name":"large_query_threshold", - "value":"0ms", - "comment":"disable large query detection for KV mode" + "name": "large_query_threshold", + "value": "0ms", + "comment": "disable large query detection for KV mode" } ], "tenant": [ @@ -91,16 +130,31 @@ "parameters": { "cluster": [ { - "name":"large_query_threshold", - "value":"600s", - "comment":"AP query exist in HTAP workload, we need it running fast too." + "name": "large_query_threshold", + "value": "600s", + "comment": "AP query exist in HTAP workload, we need it running fast too." + }, + { + "name": "enable_syslog_recycle", + "value": 1, + "comment": "enable syslog auto recycle can prevent log files from filling up disk space" + }, + { + "name": "max_syslog_file_count", + "value": 300, + "comment": "when enable_syslog_recycle is enabled, should set this value to a proper value. 300 is an empirical value." } ], "tenant": [ { - "name":"_rowsets_max_rows", + "name": "_rowsets_max_rows", "value": 32, - "comment":"for classic HTAP workloads, rowset 32 is tradeoff" + "comment": "for classic HTAP workloads, rowset 32 is tradeoff" + }, + { + "name": "log_transport_compress_all", + "value": 1, + "comment": "In scenarios with limited bandwidth, network bandwidth can be saved with a small amount of CPU overhead through RPC compression" } ] } diff --git a/src/share/redolog/ob_log_file_handler.cpp b/src/share/redolog/ob_log_file_handler.cpp index c228a58ec0..6023df2d10 100644 --- a/src/share/redolog/ob_log_file_handler.cpp +++ b/src/share/redolog/ob_log_file_handler.cpp @@ -307,7 +307,7 @@ int ObLogFileHandler::unlink(const char* file_path) while (OB_SUCC(ret)) { if (OB_FAIL(THE_IO_DEVICE->unlink(file_path)) && OB_NO_SUCH_FILE_OR_DIRECTORY != ret) { LOG_WARN("unlink failed", K(ret), K(file_path)); - ob_usleep(static_cast(UNLINK_RETRY_INTERVAL_US)); + ob_usleep(UNLINK_RETRY_INTERVAL_US); ret = OB_SUCCESS; } else if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret) { ret = OB_SUCCESS; @@ -358,7 +358,7 @@ int ObLogFileHandler::normal_retry_write(void *buf, int64_t size, int64_t offset if (REACH_TIME_INTERVAL(LOG_INTERVAL_US)) { LOG_WARN("fail to aio_write", K(ret), K(io_info), K(retry_cnt)); } else { - ob_usleep(static_cast(SLEEP_TIME_US)); + ob_usleep(SLEEP_TIME_US); } } } while (OB_FAIL(ret)); @@ -381,7 +381,7 @@ int ObLogFileHandler::open(const char *file_path, const int flags, const mode_t LOG_WARN("failed to open file", K(ret), K(file_path), K(errno), KERRMSG); if (OB_TIMEOUT == ret || OB_EAGAIN == ret) { ret = OB_SUCCESS; - ob_usleep(static_cast(ObLogDefinition::RETRY_SLEEP_TIME_IN_US)); + ob_usleep(ObLogDefinition::RETRY_SLEEP_TIME_IN_US); } } else { break; diff --git a/src/share/scheduler/ob_tenant_dag_scheduler.h b/src/share/scheduler/ob_tenant_dag_scheduler.h index c35b46aaa3..b4f63e3973 100644 --- a/src/share/scheduler/ob_tenant_dag_scheduler.h +++ b/src/share/scheduler/ob_tenant_dag_scheduler.h @@ -356,7 +356,7 @@ public: bool is_dag_failed() const { return ObIDag::DAG_STATUS_NODE_FAILED == dag_status_; } void set_add_time() { add_time_ = ObTimeUtility::fast_current_time(); } int64_t get_add_time() const { return add_time_; } - int64_t get_priority() const { return priority_; } + ObDagPrio::ObDagPrioEnum get_priority() const { return priority_; } void set_priority(ObDagPrio::ObDagPrioEnum prio) { priority_ = prio; } const ObDagId &get_dag_id() const { return id_; } int set_dag_id(const ObDagId &dag_id); @@ -1062,6 +1062,8 @@ public: template int alloc_dag(T *&dag); template + int alloc_dag_with_priority(const ObDagPrio::ObDagPrioEnum &prio, T *&dag); + template int create_and_add_dag_net(const ObIDagInitParam *param); void free_dag(ObIDag &dag); void inner_free_dag(ObIDag &dag); @@ -1297,6 +1299,27 @@ int ObTenantDagScheduler::alloc_dag(T *&dag) return ret; } +template +int ObTenantDagScheduler::alloc_dag_with_priority( + const ObDagPrio::ObDagPrioEnum &prio, T *&dag) +{ + int ret = OB_SUCCESS; + dag = NULL; + if (prio < ObDagPrio::DAG_PRIO_COMPACTION_HIGH + || prio >= ObDagPrio::DAG_PRIO_MAX) { + ret = OB_INVALID_ARGUMENT; + COMMON_LOG(WARN, "get invalid arg", K(ret), K(prio)); + } else if (OB_FAIL(alloc_dag(dag))) { + COMMON_LOG(WARN, "failed to alloc dag", K(ret)); + } else if (OB_ISNULL(dag)) { + ret = OB_ERR_UNEXPECTED; + COMMON_LOG(WARN, "dag should not be null", K(ret), KP(dag)); + } else { + dag->set_priority(prio); + } + return ret; +} + template void ObTenantDagScheduler::free_dag_net(T *&dag_net) { diff --git a/src/share/schema/ob_schema_printer.cpp b/src/share/schema/ob_schema_printer.cpp index bcf8be5a3f..378e8e476e 100644 --- a/src/share/schema/ob_schema_printer.cpp +++ b/src/share/schema/ob_schema_printer.cpp @@ -1849,7 +1849,8 @@ int ObSchemaPrinter::print_table_definition_table_options(const ObTableSchema &t if (OB_SUCC(ret) && !strict_compat_ && ObDuplicateScopeChecker::is_valid_replicate_scope(table_schema.get_duplicate_scope()) - && !is_no_table_options(sql_mode)) { + && !is_no_table_options(sql_mode) + && table_schema.is_user_table()) { // 目前只支持cluster if (table_schema.get_duplicate_scope() == ObDuplicateScope::DUPLICATE_SCOPE_CLUSTER) { if (OB_FAIL(databuff_printf(buf, buf_len, pos, "DUPLICATE_SCOPE = 'CLUSTER' "))) { diff --git a/src/share/schema/ob_schema_utils.cpp b/src/share/schema/ob_schema_utils.cpp index c051ae113b..f54c3dabc4 100644 --- a/src/share/schema/ob_schema_utils.cpp +++ b/src/share/schema/ob_schema_utils.cpp @@ -1231,6 +1231,17 @@ int ObSchemaUtils::is_drop_column_only(const AlterTableSchema &alter_table_schem return ret; } +bool ObSchemaUtils::can_add_column_group(const ObTableSchema &table_schema) +{ + bool can_add_cg = false; + if (table_schema.is_user_table() + || table_schema.is_tmp_table() + || table_schema.is_index_table()) { + can_add_cg = true; + } + return can_add_cg; +} + } // end schema } // end share } // end oceanbase diff --git a/src/share/schema/ob_schema_utils.h b/src/share/schema/ob_schema_utils.h index 6f7e8d50fb..0acc2129d2 100644 --- a/src/share/schema/ob_schema_utils.h +++ b/src/share/schema/ob_schema_utils.h @@ -156,6 +156,7 @@ public: static int mock_default_cg( const uint64_t tenant_id, share::schema::ObTableSchema &new_table_schema); + static bool can_add_column_group(const ObTableSchema &table_schema); // Optimized method to batch get latest table schemas from cache or inner_table automatically. // diff --git a/src/share/schema/ob_table_schema.cpp b/src/share/schema/ob_table_schema.cpp index bd7b7971ab..34340acb1f 100644 --- a/src/share/schema/ob_table_schema.cpp +++ b/src/share/schema/ob_table_schema.cpp @@ -8690,6 +8690,10 @@ int ObTableSchema::add_column_group(const ObColumnGroupSchema &other) if (!other.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(other)); + } else if (other.get_column_group_type() != ObColumnGroupType::DEFAULT_COLUMN_GROUP + && !ObSchemaUtils::can_add_column_group(*this)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("only default column group is allowded to add to not user/tmp table", K(ret), K(other), KPC(this)); } else if (OB_FAIL(do_add_column_group(other))) { LOG_WARN("fail to do add column group", KR(ret), K(other)); } diff --git a/src/share/schema/ob_table_schema.h b/src/share/schema/ob_table_schema.h index 9b84d9ee9e..03e41bce85 100644 --- a/src/share/schema/ob_table_schema.h +++ b/src/share/schema/ob_table_schema.h @@ -259,6 +259,12 @@ enum ObMVOnQueryComputationFlag IS_MV_ON_QUERY_COMPUTATION = 1, }; +enum ObDDLIgnoreSyncCdcFlag +{ + DO_SYNC_LOG_FOR_CDC = 0, + DONT_SYNC_LOG_FOR_CDC = 1, +}; + struct ObTableMode { OB_UNIS_VERSION_V(1); private: @@ -288,7 +294,9 @@ private: static const int32_t TM_MV_ENABLE_QUERY_REWRITE_BITS = 1; static const int32_t TM_MV_ON_QUERY_COMPUTATION_OFFSET = 28; static const int32_t TM_MV_ON_QUERY_COMPUTATION_BITS = 1; - static const int32_t TM_RESERVED = 3; + static const int32_t TM_DDL_IGNORE_SYNC_CDC_OFFSET = 29; + static const int32_t TM_DDL_IGNORE_SYNC_CDC_BITS = 1; + static const int32_t TM_RESERVED = 2; static const uint32_t MODE_FLAG_MASK = (1U << TM_MODE_FLAG_BITS) - 1; static const uint32_t PK_MODE_MASK = (1U << TM_PK_MODE_BITS) - 1; @@ -303,6 +311,7 @@ private: static const uint32_t TABLE_REFERENCED_BY_MV_MASK = (1U << TM_TABLE_REFERENCED_BY_MV_BITS) - 1; static const uint32_t MV_ENABLE_QUERY_REWRITE_MASK = (1U << TM_MV_ENABLE_QUERY_REWRITE_BITS) - 1; static const uint32_t MV_ON_QUERY_COMPUTATION_MASK = (1U << TM_MV_ON_QUERY_COMPUTATION_BITS) - 1; + static const uint32_t DDL_IGNORE_SYNC_CDC_MASK = (1U << TM_DDL_IGNORE_SYNC_CDC_BITS) - 1; public: ObTableMode() { reset(); } virtual ~ObTableMode() { reset(); } @@ -381,7 +390,8 @@ public: "mv_available_flag", mv_available_flag_, "table_referenced_by_mv_flag", table_referenced_by_mv_flag_, "mv_enable_query_rewrite_flag", mv_enable_query_rewrite_flag_, - "mv_on_query_computation_flag", mv_on_query_computation_flag_); + "mv_on_query_computation_flag", mv_on_query_computation_flag_, + "ddl_table_ignore_sync_cdc_flag", ddl_table_ignore_sync_cdc_flag_); union { int32_t mode_; struct { @@ -398,6 +408,7 @@ public: uint32_t table_referenced_by_mv_flag_ : TM_TABLE_REFERENCED_BY_MV_BITS; uint32_t mv_enable_query_rewrite_flag_ : TM_MV_ENABLE_QUERY_REWRITE_BITS; uint32_t mv_on_query_computation_flag_ : TM_MV_ON_QUERY_COMPUTATION_BITS; + uint32_t ddl_table_ignore_sync_cdc_flag_ : TM_DDL_IGNORE_SYNC_CDC_BITS; uint32_t reserved_ :TM_RESERVED; }; }; @@ -723,6 +734,10 @@ public: { return IS_MV_ON_QUERY_COMPUTATION == (enum ObMVOnQueryComputationFlag)table_mode_.mv_on_query_computation_flag_; } inline void set_mv_on_query_computation(const ObMVOnQueryComputationFlag flag) { table_mode_.mv_on_query_computation_flag_ = flag; } + inline void set_ddl_ignore_sync_cdc_flag(const ObDDLIgnoreSyncCdcFlag flag) + { table_mode_.ddl_table_ignore_sync_cdc_flag_ = flag; } + inline bool is_ddl_table_ignored_to_sync_cdc() const + { return DONT_SYNC_LOG_FOR_CDC == table_mode_.ddl_table_ignore_sync_cdc_flag_; } inline void set_session_id(const uint64_t id) { session_id_ = id; } inline uint64_t get_session_id() const { return session_id_; } diff --git a/src/share/stat/ob_basic_stats_estimator.cpp b/src/share/stat/ob_basic_stats_estimator.cpp index 32e8baeb29..3df0ce6746 100644 --- a/src/share/stat/ob_basic_stats_estimator.cpp +++ b/src/share/stat/ob_basic_stats_estimator.cpp @@ -93,6 +93,9 @@ int ObBasicStatsEstimator::estimate(const ObOptStatGatherParam ¶m, LOG_WARN("failed to add group by info", K(ret)); } else if (OB_FAIL(add_stat_item(ObPartitionId(src_tab_stat, calc_part_id_str, -1)))) { LOG_WARN("failed to add partition id", K(ret)); + } else if (param.is_specify_partition_ && + OB_FAIL(fill_partition_info(allocator, param.partition_infos_))) { + LOG_WARN("failed to add partition info", K(ret)); } } else if (OB_UNLIKELY(param.partition_infos_.count() > 1) || OB_ISNULL(dst_opt_stats.at(0).table_stat_)) { @@ -200,6 +203,8 @@ int ObBasicStatsEstimator::estimate_block_count(ObExecContext &ctx, block_num_stat = new (buf) BlockNumStat(); block_num_stat->tab_macro_cnt_ = estimate_result.at(i).macro_block_count_; block_num_stat->tab_micro_cnt_ = estimate_result.at(i).micro_block_count_; + block_num_stat->sstable_row_cnt_ = estimate_result.at(i).sstable_row_count_; + block_num_stat->memtable_row_cnt_ = estimate_result.at(i).memtable_row_count_; total_sstable_row_cnt += estimate_result.at(i).sstable_row_count_; total_memtable_row_cnt += estimate_result.at(i).memtable_row_count_; int64_t partition_id = static_cast(estimate_result.at(i).part_id_); @@ -213,7 +218,9 @@ int ObBasicStatsEstimator::estimate_block_count(ObExecContext &ctx, block_num_stat->tab_macro_cnt_, block_num_stat->tab_micro_cnt_, block_num_stat->cg_macro_cnt_arr_, - block_num_stat->cg_micro_cnt_arr_))) { + block_num_stat->cg_micro_cnt_arr_, + block_num_stat->sstable_row_cnt_, + block_num_stat->memtable_row_cnt_))) { LOG_WARN("faild to add", K(ret)); } } else if (param.part_level_ == share::schema::PARTITION_LEVEL_TWO) { @@ -226,7 +233,9 @@ int ObBasicStatsEstimator::estimate_block_count(ObExecContext &ctx, block_num_stat->tab_macro_cnt_, block_num_stat->tab_micro_cnt_, block_num_stat->cg_macro_cnt_arr_, - block_num_stat->cg_micro_cnt_arr_))) { + block_num_stat->cg_micro_cnt_arr_, + block_num_stat->sstable_row_cnt_, + block_num_stat->memtable_row_cnt_))) { LOG_WARN("faild to add", K(ret)); } else { int64_t idx = 0; @@ -240,7 +249,9 @@ int ObBasicStatsEstimator::estimate_block_count(ObExecContext &ctx, block_num_stat->tab_macro_cnt_, block_num_stat->tab_micro_cnt_, block_num_stat->cg_macro_cnt_arr_, - block_num_stat->cg_micro_cnt_arr_))) { + block_num_stat->cg_micro_cnt_arr_, + block_num_stat->sstable_row_cnt_, + block_num_stat->memtable_row_cnt_))) { LOG_WARN("faild to add", K(ret)); } } @@ -271,6 +282,8 @@ int ObBasicStatsEstimator::estimate_block_count(ObExecContext &ctx, block_num_stat = new (buf) BlockNumStat(); block_num_stat->tab_macro_cnt_ = global_tab_stat.get_macro_block_count(); block_num_stat->tab_micro_cnt_ = global_tab_stat.get_micro_block_count(); + block_num_stat->sstable_row_cnt_ = global_tab_stat.get_sstable_row_cnt(); + block_num_stat->memtable_row_cnt_ = global_tab_stat.get_memtable_row_cnt(); if (OB_FAIL(block_num_stat->cg_macro_cnt_arr_.assign(global_tab_stat.get_cg_macro_arr())) || OB_FAIL(block_num_stat->cg_micro_cnt_arr_.assign(global_tab_stat.get_cg_micro_arr()))) { LOG_WARN("failed to assign", K(ret)); @@ -289,6 +302,8 @@ int ObBasicStatsEstimator::estimate_block_count(ObExecContext &ctx, block_num_stat = new (buf) BlockNumStat(); block_num_stat->tab_macro_cnt_ = first_part_tab_stats.at(i).get_macro_block_count(); block_num_stat->tab_micro_cnt_ = first_part_tab_stats.at(i).get_micro_block_count(); + block_num_stat->sstable_row_cnt_ = first_part_tab_stats.at(i).get_sstable_row_cnt(); + block_num_stat->memtable_row_cnt_ = first_part_tab_stats.at(i).get_memtable_row_cnt(); if (OB_FAIL(block_num_stat->cg_macro_cnt_arr_.assign(first_part_tab_stats.at(i).get_cg_macro_arr())) || OB_FAIL(block_num_stat->cg_micro_cnt_arr_.assign(first_part_tab_stats.at(i).get_cg_micro_arr()))) { LOG_WARN("failed to assign", K(ret)); @@ -783,7 +798,7 @@ int ObBasicStatsEstimator::estimate_stale_partition(ObExecContext &ctx, cur_part_id = dst_part_id; cur_inc_mod_count = inc_mod_count; } else if (OB_FAIL(check_partition_stat_state(cur_part_id, - has_subpart_invalid_inc ? 0 : cur_inc_mod_count, + has_subpart_invalid_inc ? -1 : cur_inc_mod_count, stale_percent_threshold, partition_stat_infos))) { LOG_WARN("failed to check partition stat state", K(ret)); @@ -804,13 +819,13 @@ int ObBasicStatsEstimator::estimate_stale_partition(ObExecContext &ctx, ret = OB_SUCCESS; if (cur_part_id != -1 && OB_FAIL(check_partition_stat_state(cur_part_id, - has_subpart_invalid_inc ? 0 : cur_inc_mod_count, + has_subpart_invalid_inc ? -1 : cur_inc_mod_count, stale_percent_threshold, partition_stat_infos))) { LOG_WARN("failed to check partition stat state", K(ret)); } else if (is_check_global && OB_FAIL(check_partition_stat_state(global_part_id, - has_part_invalid_inc ? 0 : table_inc_modified, + has_part_invalid_inc ? -1 : table_inc_modified, stale_percent_threshold, partition_stat_infos))) { LOG_WARN("failed to check partition stat state", K(ret)); @@ -826,6 +841,7 @@ int ObBasicStatsEstimator::estimate_stale_partition(ObExecContext &ctx, } } } + ObSEArray record_first_part_ids; for (int64_t i = 0; OB_SUCC(ret) && i < partition_infos.count(); ++i) { int64_t partition_id = partition_infos.at(i).part_id_; int64_t first_part_id = partition_infos.at(i).first_part_id_; @@ -836,16 +852,22 @@ int ObBasicStatsEstimator::estimate_stale_partition(ObExecContext &ctx, LOG_WARN("failed to push back", K(ret)); } else {/*do nothing*/} } - if (first_part_id != OB_INVALID_ID && !is_contain(monitor_modified_part_ids, first_part_id)) { + if (OB_SUCC(ret) && + first_part_id != OB_INVALID_ID && + !is_contain(monitor_modified_part_ids, first_part_id) && + !is_contain(record_first_part_ids, first_part_id)) { ObPartitionStatInfo partition_stat_info(first_part_id, 0, false, true); - ret = partition_stat_infos.push_back(partition_stat_info); + if (OB_FAIL(partition_stat_infos.push_back(partition_stat_info)) || + OB_FAIL(record_first_part_ids.push_back(first_part_id))) { + LOG_WARN("failed to push back", K(ret)); + } } } } - LOG_INFO("succeed to estimate stale partition", K(stale_percent_threshold), - K(partition_stat_infos), - K(partition_infos), - K(monitor_modified_part_ids)); + LOG_TRACE("succeed to estimate stale partition", K(stale_percent_threshold), + K(partition_stat_infos), + K(partition_infos), + K(monitor_modified_part_ids)); return ret; } @@ -881,6 +903,10 @@ int ObBasicStatsEstimator::update_last_modified_count(sqlclient::ObISQLConnectio ObSqlString tablet_list; int64_t affected_rows = 0; bool is_valid = true; + bool is_all_update = false; + //if this is virtual table real agent, we need update the real table id modifed count + uint64_t table_id = share::is_oracle_mapping_real_virtual_table(param.table_id_) ? + share::get_real_table_mappings_tid(param.table_id_) : param.table_id_; if (OB_ISNULL(conn)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(conn)); @@ -888,17 +914,18 @@ int ObBasicStatsEstimator::update_last_modified_count(sqlclient::ObISQLConnectio LOG_WARN("failed to check table read write valid", K(ret)); } else if (!is_valid) { // do nothing - } else if (OB_FAIL(gen_tablet_list(param, tablet_list))) { + } else if (OB_FAIL(gen_tablet_list(param, tablet_list, is_all_update))) { LOG_WARN("failed to gen partition list", K(ret)); - } else if (tablet_list.empty()) { + } else if (tablet_list.empty() && !is_all_update) { /*do nothing*/ } else if (OB_FAIL(udpate_sql.append_fmt( "update %s set last_inserts = inserts, last_updates = updates, last_deletes = deletes " \ - "where tenant_id = %lu and table_id = %lu and tablet_id in %s;", + "where tenant_id = %lu and table_id = %lu %s %s;", share::OB_ALL_MONITOR_MODIFIED_TNAME, share::schema::ObSchemaUtils::get_extract_tenant_id(param.tenant_id_, param.tenant_id_), - share::schema::ObSchemaUtils::get_extract_schema_id(param.tenant_id_, param.table_id_), - tablet_list.ptr()))) { + share::schema::ObSchemaUtils::get_extract_schema_id(param.tenant_id_, table_id), + !tablet_list.empty() ? "and tablet_id in" : " ", + !tablet_list.empty() ? tablet_list.ptr() : " "))) { LOG_WARN("failed to append fmt", K(ret)); } else if (OB_FAIL(conn->execute_write(param.tenant_id_, udpate_sql.ptr(), affected_rows))) { LOG_WARN("failed to execute sql", K(ret), K(udpate_sql)); @@ -1001,49 +1028,57 @@ int ObBasicStatsEstimator::check_partition_stat_state(const int64_t partition_id for (int64_t i = 0; !find_it && i < partition_stat_infos.count(); ++i) { if (partition_stat_infos.at(i).partition_id_ == partition_id) { //locked partition id or no arrived stale percent threshold no need regather stats. - double stale_percent = partition_stat_infos.at(i).row_cnt_ <= 0 ? 1.0 : - 1.0 * inc_mod_count / partition_stat_infos.at(i).row_cnt_; + double stale_percent = 0.0; + if (inc_mod_count < 0 || partition_stat_infos.at(i).row_cnt_ <= 0) { + stale_percent = inc_mod_count == 0 ? 0.0 : 1.0; + } else { + stale_percent = 1.0 * inc_mod_count / partition_stat_infos.at(i).row_cnt_; + } partition_stat_infos.at(i).is_no_stale_ = stale_percent <= stale_percent_threshold; find_it = true; } } if (!find_it) { ObPartitionStatInfo partition_stat_info(partition_id, 0, false, false); - partition_stat_info.is_no_stale_ = true; + partition_stat_info.is_no_stale_ = false; ret = partition_stat_infos.push_back(partition_stat_info); } return ret; } int ObBasicStatsEstimator::gen_tablet_list(const ObTableStatParam ¶m, - ObSqlString &tablet_list) + ObSqlString &tablet_list, + bool &is_all_update) { int ret = OB_SUCCESS; ObSEArray tablet_ids; + is_all_update = false; if (param.global_stat_param_.need_modify_) { - if (param.part_level_ == share::schema::ObPartitionLevel::PARTITION_LEVEL_ZERO) { - if (OB_UNLIKELY(param.global_tablet_id_ == ObTabletID::INVALID_TABLET_ID)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(param)); - } else if (OB_FAIL(tablet_ids.push_back(param.global_tablet_id_))) { - LOG_WARN("failed to push back", K(ret)); + if (param.part_level_ == share::schema::ObPartitionLevel::PARTITION_LEVEL_ZERO || + !param.global_stat_param_.gather_approx_) { + is_all_update = true; + } + } + if (OB_SUCC(ret) && !is_all_update && param.part_stat_param_.need_modify_) { + if (param.part_level_ == share::schema::ObPartitionLevel::PARTITION_LEVEL_ONE) { + for (int64_t i = 0; OB_SUCC(ret) && i < param.part_infos_.count(); ++i) { + if (OB_FAIL(tablet_ids.push_back(param.part_infos_.at(i).tablet_id_.id()))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } else if (param.part_level_ == share::schema::ObPartitionLevel::PARTITION_LEVEL_TWO) { + for (int64_t i = 0; OB_SUCC(ret) && i < param.part_infos_.count(); ++i) { + for (int64_t j = 0; OB_SUCC(ret) && j < param.subpart_infos_.count(); ++j) { + if (param.part_infos_.at(i).part_id_ == param.subpart_infos_.at(j).first_part_id_) { + if (OB_FAIL(tablet_ids.push_back(param.subpart_infos_.at(j).tablet_id_.id()))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } } } } - if (OB_SUCC(ret) && param.part_stat_param_.need_modify_ && - param.part_level_ != share::schema::ObPartitionLevel::PARTITION_LEVEL_TWO) { - for (int64_t i = 0; OB_SUCC(ret) && i < param.part_infos_.count(); ++i) { - if (OB_FAIL(tablet_ids.push_back(param.part_infos_.at(i).tablet_id_.id()))) { - LOG_WARN("failed to push back", K(ret)); - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < param.approx_part_infos_.count(); ++i) { - if (OB_FAIL(tablet_ids.push_back(param.approx_part_infos_.at(i).tablet_id_.id()))) { - LOG_WARN("failed to push back", K(ret)); - } - } - } - if (OB_SUCC(ret) && param.subpart_stat_param_.need_modify_) { + if (OB_SUCC(ret) && !is_all_update && param.subpart_stat_param_.need_modify_) { for (int64_t i = 0; OB_SUCC(ret) && i < param.subpart_infos_.count(); ++i) { if (OB_FAIL(tablet_ids.push_back(param.subpart_infos_.at(i).tablet_id_.id()))) { LOG_WARN("failed to push back", K(ret)); @@ -1062,7 +1097,6 @@ int ObBasicStatsEstimator::gen_tablet_list(const ObTableStatParam ¶m, return ret; } - int ObBasicStatsEstimator::get_all_tablet_id_and_object_id(const ObTableStatParam ¶m, ObIArray &tablet_ids, ObIArray &partition_ids) @@ -1095,81 +1129,39 @@ int ObBasicStatsEstimator::get_all_tablet_id_and_object_id(const ObTableStatPara return ret; } -int ObBasicStatsEstimator::get_need_stats_table_cnt(ObExecContext &ctx, - const int64_t tenant_id, - int64_t &task_table_count) -{ - int ret = OB_SUCCESS; - ObSqlString select_sql; - if (OB_FAIL(select_sql.append_fmt( - "select count(1) as cnt from (select distinct m.table_id from " \ - "%s m left join %s up on m.table_id = up.table_id and up.pname = 'STALE_PERCENT' join %s gp on gp.sname = 'STALE_PERCENT' " \ - "where (case when (m.inserts+m.updates+m.deletes) = 0 then 0 " - "else ((m.inserts+m.updates+m.deletes) - (m.last_inserts+m.last_updates+m.last_deletes)) * 1.0 / (m.inserts+m.updates+m.deletes) > " \ - "(CASE WHEN up.valchar IS NOT NULL THEN cast(up.valchar as signed) * 1.0 / 100 ELSE Cast(gp.spare4 AS signed) * 1.0 / 100 end) end) " \ - "UNION select distinct table_id from %s where table_id not in (select table_id from %s)) ", - share::OB_ALL_MONITOR_MODIFIED_TNAME, - share::OB_ALL_OPTSTAT_USER_PREFS_TNAME, - share::OB_ALL_OPTSTAT_GLOBAL_PREFS_TNAME, - share::OB_ALL_MONITOR_MODIFIED_TNAME, - share::OB_ALL_TABLE_STAT_TNAME))) { - LOG_WARN("failed to append fmt", K(ret)); - } else { - ObCommonSqlProxy *sql_proxy = ctx.get_sql_proxy(); - SMART_VAR(ObMySQLProxy::MySQLResult, proxy_result) { - sqlclient::ObMySQLResult *client_result = NULL; - ObSQLClientRetryWeak sql_client_retry_weak(sql_proxy); - if (OB_FAIL(sql_client_retry_weak.read(proxy_result, tenant_id, select_sql.ptr()))) { - LOG_WARN("failed to execute sql", K(ret), K(select_sql)); - } else if (OB_ISNULL(client_result = proxy_result.get_result())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to execute sql", K(ret)); - } else { - while (OB_SUCC(ret) && OB_SUCC(client_result->next())) { - int64_t idx = 0; - ObObj obj; - if (OB_FAIL(client_result->get_obj(idx, obj))) { - LOG_WARN("failed to get object", K(ret)); - } else if (OB_FAIL(obj.get_int(task_table_count))) { - LOG_WARN("failed to get int", K(ret), K(obj)); - } - } - ret = OB_ITER_END == ret ? OB_SUCCESS : ret; - } - int tmp_ret = OB_SUCCESS; - if (NULL != client_result) { - if (OB_SUCCESS != (tmp_ret = client_result->close())) { - LOG_WARN("close result set failed", K(ret), K(tmp_ret)); - ret = COVER_SUCC(tmp_ret); - } - } - } - LOG_TRACE("succeed to get table count that need gathering table stats", K(ret), K(task_table_count)); - } - return ret; -} - int ObBasicStatsEstimator::get_need_stats_tables(ObExecContext &ctx, const int64_t tenant_id, - ObIArray &table_ids, - int64_t &slice_cnt) + const int64_t offset, + const int64_t slice_cnt, + ObIArray &table_ids) { int ret = OB_SUCCESS; + ObSqlString gather_table_type_list; ObSqlString select_sql; - if (OB_FAIL(select_sql.append_fmt( - "select distinct table_id from (select m.table_id from " \ - "%s m left join %s up on m.table_id = up.table_id and up.pname = 'STALE_PERCENT' join %s gp on gp.sname = 'STALE_PERCENT' " \ - "where (case when (m.inserts+m.updates+m.deletes) = 0 then 0 "\ - "else ((m.inserts+m.updates+m.deletes) - (m.last_inserts+m.last_updates+m.last_deletes)) * 1.0 / (m.inserts+m.updates+m.deletes) > " \ - "(CASE WHEN up.valchar IS NOT NULL THEN cast(up.valchar as signed) * 1.0 / 100 ELSE Cast(gp.spare4 AS signed) * 1.0 / 100 end) end) "\ - " UNION ALL select table_id from %s where table_id not in (select table_id from %s)) " - "ORDER BY table_id DESC limit %ld", - share::OB_ALL_MONITOR_MODIFIED_TNAME, - share::OB_ALL_OPTSTAT_USER_PREFS_TNAME, - share::OB_ALL_OPTSTAT_GLOBAL_PREFS_TNAME, - share::OB_ALL_MONITOR_MODIFIED_TNAME, - share::OB_ALL_TABLE_STAT_TNAME, - slice_cnt))) { + if (OB_FAIL(get_gather_table_type_list(gather_table_type_list))) { + LOG_WARN("failed to get gather table type list", K(ret)); + } else if (OB_FAIL(select_sql.append_fmt("SELECT /*+no_rewrite*/table_id "\ + "FROM (SELECT tenant_id,"\ + " table_id,"\ + " table_type"\ + " FROM %s"\ + " WHERE table_type IN %s"\ + " ORDER BY tenant_id,"\ + " table_id"\ + " LIMIT %ld, %ld) t "\ + "WHERE table_type = %u "\ + " OR EXISTS(SELECT 1 "\ + " FROM %s m"\ + " WHERE t.table_id = m.table_id"\ + " AND t.tenant_id = m.tenant_id"\ + " AND inserts + deletes + updates > 0"\ + " limit 1); ", + share::OB_ALL_TABLE_TNAME, + gather_table_type_list.ptr(), + offset, + slice_cnt, + share::schema::ObTableType::VIRTUAL_TABLE, + share::OB_ALL_MONITOR_MODIFIED_TNAME))) { LOG_WARN("failed to append fmt", K(ret)); } else { ObCommonSqlProxy *sql_proxy = ctx.get_sql_proxy(); @@ -1205,7 +1197,7 @@ int ObBasicStatsEstimator::get_need_stats_tables(ObExecContext &ctx, } } LOG_TRACE("succeed to get table ids that need gathering table stats", - K(ret), K(slice_cnt), K(tenant_id), K(table_ids.count()), K(table_ids)); + K(select_sql), K(offset), K(slice_cnt), K(table_ids)); } return ret; } @@ -1400,5 +1392,23 @@ int ObBasicStatsEstimator::check_can_use_column_store_and_split_part_gather(cons return ret; } +int ObBasicStatsEstimator::get_gather_table_type_list(ObSqlString &gather_table_type_list) +{ + int ret = OB_SUCCESS; + int64_t table_type_arr[] = {share::schema::ObTableType::SYSTEM_TABLE, + share::schema::ObTableType::VIRTUAL_TABLE, + share::schema::ObTableType::USER_TABLE, + share::schema::ObTableType::EXTERNAL_TABLE}; + int64_t table_type_cnt = sizeof(table_type_arr)/sizeof(table_type_arr[0]); + for (int64_t i = 0; OB_SUCC(ret) && i < table_type_cnt; ++i) { + char prefix = (i == 0 ? '(' : ' '); + char suffix = (i == table_type_cnt - 1 ? ')' : ','); + if (OB_FAIL(gather_table_type_list.append_fmt("%c%lu%c", prefix, table_type_arr[i], suffix))) { + LOG_WARN("failed to append sql", K(ret)); + } else {/*do nothing*/} + } + return ret; +} + } // end of common } // end of oceanbase diff --git a/src/share/stat/ob_basic_stats_estimator.h b/src/share/stat/ob_basic_stats_estimator.h index 83df8808b4..a5c32d22e3 100644 --- a/src/share/stat/ob_basic_stats_estimator.h +++ b/src/share/stat/ob_basic_stats_estimator.h @@ -102,7 +102,8 @@ public: ObIArray &partition_stat_infos); static int gen_tablet_list(const ObTableStatParam ¶m, - ObSqlString &tablet_list); + ObSqlString &tablet_list, + bool &is_all_update); static int do_estimate_block_count(ObExecContext &ctx, const uint64_t tenant_id, @@ -137,12 +138,9 @@ public: static int get_need_stats_tables(ObExecContext &ctx, const int64_t tenant_id, - ObIArray &table_ids, - int64_t &slice_cnt); - - static int get_need_stats_table_cnt(ObExecContext &ctx, - const int64_t tenant_id, - int64_t &task_table_count); + const int64_t offset, + const int64_t slice_cnt, + ObIArray &table_ids); int estimate(const ObOptStatGatherParam ¶m, ObIArray &dst_opt_stats); @@ -180,6 +178,8 @@ private: const int64_t degree, bool &use_column_store, bool &use_split_part); + + static int get_gather_table_type_list(ObSqlString &gather_table_type_list); }; } diff --git a/src/share/stat/ob_dbms_stats_copy_table_stats.cpp b/src/share/stat/ob_dbms_stats_copy_table_stats.cpp index b44670fa83..fd472601e5 100644 --- a/src/share/stat/ob_dbms_stats_copy_table_stats.cpp +++ b/src/share/stat/ob_dbms_stats_copy_table_stats.cpp @@ -16,6 +16,7 @@ #include "share/stat/ob_opt_column_stat.h" #include "share/stat/ob_dbms_stats_utils.h" #include "share/stat/ob_dbms_stats_copy_table_stats.h" +#include "share/stat/ob_dbms_stats_history_manager.h" int CopyTableStatHelper::copy_part_stat(ObIArray &table_stats) { @@ -419,8 +420,28 @@ int ObDbmsStatsCopyTableStats::copy_tab_col_stats(sql::ObExecContext &ctx, LOG_WARN("src table stat is not analyzed", K(table_stat_param.part_infos_.at(0).part_id_)); } else if (OB_FAIL(copy_stat_helper.copy_part_col_stat(table_stat_param.is_subpart_name_, col_handles, table_stats, column_stats))) { LOG_WARN("failed to copy table column stat", K(ret), KPC(copy_stat_helper.src_part_stat_)); - } else if (OB_FAIL(ObDbmsStatsUtils::split_batch_write(ctx, table_stats, column_stats))) { - LOG_WARN("failed to split batch write stat", K(ret)); + } + if (OB_SUCC(ret)) { + ObMySQLTransaction trans; + //begin trans + if (OB_FAIL(trans.start(ctx.get_sql_proxy(), table_stat_param.tenant_id_))) { + LOG_WARN("fail to start transaction", K(ret)); + } else if (OB_FAIL(ObDbmsStatsHistoryManager::backup_opt_stats(ctx, trans, table_stat_param, ObTimeUtility::current_time()))) { + LOG_WARN("failed to backup opt stats", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::split_batch_write(ctx, trans.get_connection(), table_stats, column_stats))) { + LOG_WARN("failed to split batch write", K(ret)); + } else {/*do nothing*/} + //end trans + if (OB_SUCC(ret)) { + if (OB_FAIL(trans.end(true))) { + LOG_WARN("fail to commit transaction", K(ret)); + } + } else { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(false))) { + LOG_WARN("fail to roll back transaction", K(tmp_ret)); + } + } } return ret; } diff --git a/src/share/stat/ob_dbms_stats_executor.cpp b/src/share/stat/ob_dbms_stats_executor.cpp index 07b063f6f0..9ece10b782 100644 --- a/src/share/stat/ob_dbms_stats_executor.cpp +++ b/src/share/stat/ob_dbms_stats_executor.cpp @@ -341,7 +341,7 @@ int ObDbmsStatsExecutor::split_gather_partition_stats(ObExecContext &ctx, } } if (OB_SUCC(ret)) { - if (gather_helper.maximum_gather_col_cnt_ >= param.column_params_.count()) { + if (gather_helper.maximum_gather_col_cnt_ >= param.get_need_gather_column()) { ObSEArray all_tstats; ObSEArray all_cstats; ObSEArray opt_stats; @@ -677,7 +677,7 @@ int ObDbmsStatsExecutor::check_need_split_gather(const ObTableStatParam ¶m, GatherHelper &gather_helper) { int ret = OB_SUCCESS; - int64_t column_cnt = param.column_params_.count(); + int64_t column_cnt = param.get_need_gather_column(); int64_t partition_cnt = param.subpart_stat_param_.need_modify_ ? param.subpart_infos_.count() : (param.part_stat_param_.need_modify_ ? param.part_infos_.count() + param.approx_part_infos_.count() : 1); bool need_histgoram = param.subpart_stat_param_.need_modify_ ? param.subpart_stat_param_.gather_histogram_ : @@ -921,7 +921,7 @@ int ObDbmsStatsExecutor::set_column_stats(ObExecContext &ctx, col_stat->set_column_id(key.column_id_); col_stat->set_collation_type(param.table_param_.column_params_.at(0).cs_type_); col_stat->set_last_analyzed(0); - if (OB_FAIL(do_set_column_stats(param, col_stat))) { + if (OB_FAIL(do_set_column_stats(*alloc, ctx.get_my_session()->get_dtc_params(), param, col_stat))) { LOG_WARN("failed to do set table stats", K(ret)); } else if (OB_FAIL(column_stats.push_back(col_stat))) { LOG_WARN("failed to push back column stat", K(ret)); @@ -994,7 +994,9 @@ int ObDbmsStatsExecutor::do_set_table_stats(const ObSetTableStatParam ¶m, return ret; } -int ObDbmsStatsExecutor::do_set_column_stats(const ObSetColumnStatParam ¶m, +int ObDbmsStatsExecutor::do_set_column_stats(ObIAllocator &allocator, + const ObDataTypeCastParams &dtc_params, + const ObSetColumnStatParam ¶m, ObOptColumnStat *&column_stat) { int ret = OB_SUCCESS; @@ -1025,9 +1027,21 @@ int ObDbmsStatsExecutor::do_set_column_stats(const ObSetColumnStatParam ¶m, if (param.avgclen_ > 0) { column_stat->set_avg_len(param.avgclen_); } - //5.set hist_param TODO @jiangxiu.wt + //5.set max/val value + if (param.hist_param_.minval_ != NULL || param.hist_param_.maxval_ != NULL) { + ObCastCtx cast_ctx(&allocator, &dtc_params, CM_NONE, param.col_meta_.get_collation_type()); + if ((param.hist_param_.minval_ != NULL && + OB_FAIL(ObObjCaster::to_type(param.col_meta_.get_type(), cast_ctx, *param.hist_param_.minval_, column_stat->get_min_value()))) || + (param.hist_param_.maxval_ != NULL && + OB_FAIL(ObObjCaster::to_type(param.col_meta_.get_type(), cast_ctx, *param.hist_param_.maxval_, column_stat->get_max_value())))) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("Invalid or inconsistent input values", K(ret), K(param)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL,"Invalid or inconsistent input values"); + } + } + //6.set hist_param TODO @jiangxiu.wt //other options support later. - LOG_TRACE("succeed to do set column stats", K(*column_stat)); + LOG_TRACE("succeed to do set column stats", K(param), K(*column_stat)); } return ret; } diff --git a/src/share/stat/ob_dbms_stats_executor.h b/src/share/stat/ob_dbms_stats_executor.h index 6530b4f19a..08ef7a92db 100644 --- a/src/share/stat/ob_dbms_stats_executor.h +++ b/src/share/stat/ob_dbms_stats_executor.h @@ -142,7 +142,9 @@ private: static int do_set_table_stats(const ObSetTableStatParam ¶m, ObOptTableStat *table_stat); - static int do_set_column_stats(const ObSetColumnStatParam ¶m, + static int do_set_column_stats(ObIAllocator &allocator, + const ObDataTypeCastParams &dtc_params, + const ObSetColumnStatParam ¶m, ObOptColumnStat *&column_stat); static int reset_table_locked_state(ObExecContext &ctx, diff --git a/src/share/stat/ob_dbms_stats_export_import.cpp b/src/share/stat/ob_dbms_stats_export_import.cpp index 7d034063bd..15fcbf2717 100644 --- a/src/share/stat/ob_dbms_stats_export_import.cpp +++ b/src/share/stat/ob_dbms_stats_export_import.cpp @@ -1030,7 +1030,7 @@ int ObDbmsStatsExportImport::get_opt_stat(ObExecContext &ctx, } else if (OB_FAIL(num_val.extract_valid_int64_with_trunc(int_val))) { LOG_WARN("extract_valid_int64_with_trunc failed", K(ret), K(num_val)); } else if (int_val > 0) { - if (OB_UNLIKELY(col_stat->get_histogram().get_density() <= 0.0)) { + if (OB_UNLIKELY(col_stat->get_histogram().get_density() < 0.0)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(result_objs), K(ret), KPC(col_stat)); } else if (col_stat->get_histogram().get_buckets().empty()) { diff --git a/src/share/stat/ob_dbms_stats_gather.cpp b/src/share/stat/ob_dbms_stats_gather.cpp index 9fe5c89929..5b4c25660c 100644 --- a/src/share/stat/ob_dbms_stats_gather.cpp +++ b/src/share/stat/ob_dbms_stats_gather.cpp @@ -95,12 +95,15 @@ int ObDbmsStatsGather::classfy_column_histogram(const ObOptStatGatherParam ¶ LOG_WARN("get unexpected error", K(ret), KPC(dst_col_stat), K(col_param)); } else if (col_param.need_basic_stat() && col_param.bucket_num_ > 1 && - dst_col_stat->get_num_distinct() > 0) { + dst_col_stat->get_num_distinct() > 0 && + dst_col_stat->get_num_not_null() > 0) { int64_t max_disuse_cnt = std::ceil(dst_col_stat->get_num_not_null() * 1.0 / col_param.bucket_num_); //After testing, the error of using hyperloglog to estimate ndv is within %5. const double MAX_LLC_NDV_ERR_RATE = !param.need_approx_ndv_ ? 0.0 : 0.05; const int64_t fault_tolerance_cnt = std::ceil(dst_col_stat->get_num_distinct() * MAX_LLC_NDV_ERR_RATE); - if (dst_col_stat->get_num_distinct() >= col_param.bucket_num_ + max_disuse_cnt + fault_tolerance_cnt) { + double sample_val = dst_col_stat->get_histogram().get_sample_size() * 100.0 / dst_col_stat->get_num_not_null(); + if (dst_col_stat->get_num_distinct() >= col_param.bucket_num_ + max_disuse_cnt + fault_tolerance_cnt || + sample_val < 100.0 * (1.0 - 1.0 / col_param.bucket_num_)) { //directly gather hybrid histogram dst_col_stat->get_histogram().set_type(ObHistType::HYBIRD); } else { @@ -179,6 +182,8 @@ int ObDbmsStatsGather::init_opt_stat(ObIAllocator &allocator, } else { tab_stat->set_macro_block_num(block_num_stat->tab_macro_cnt_); tab_stat->set_micro_block_num(block_num_stat->tab_micro_cnt_); + tab_stat->set_sstable_row_count(block_num_stat->sstable_row_cnt_); + tab_stat->set_memtable_row_count(block_num_stat->memtable_row_cnt_); } } for (int64_t i = 0; OB_SUCC(ret) && i < param.column_params_.count(); ++i) { diff --git a/src/share/stat/ob_dbms_stats_maintenance_window.cpp b/src/share/stat/ob_dbms_stats_maintenance_window.cpp index 0b57360255..d9cc26af25 100644 --- a/src/share/stat/ob_dbms_stats_maintenance_window.cpp +++ b/src/share/stat/ob_dbms_stats_maintenance_window.cpp @@ -374,7 +374,7 @@ int ObDbmsStatsMaintenanceWindow::is_stats_maintenance_window_attr(const sql::Ob if (0 == attr_name.case_compare("job_action")) { if (0 == job_name.case_compare(opt_stats_history_manager)) { const char *job_action_name = "DBMS_STATS.PURGE_STATS("; - if (0 == strncasecmp(val_name.ptr(), job_action_name, strlen(job_action_name))) { + if (!val_name.empty() && 0 == strncasecmp(val_name.ptr(), job_action_name, strlen(job_action_name))) { if (OB_FAIL(dml.add_column("job_action", ObHexEscapeSqlStr(val_name)))) { LOG_WARN("failed to add column", K(ret)); } else if (OB_FAIL(dml.add_column("what", ObHexEscapeSqlStr(val_name)))) { @@ -385,7 +385,7 @@ int ObDbmsStatsMaintenanceWindow::is_stats_maintenance_window_attr(const sql::Ob } else {/*do nothing*/} } else { const char *job_action_name = "DBMS_STATS.GATHER_DATABASE_STATS_JOB_PROC("; - if (0 == strncasecmp(val_name.ptr(), job_action_name, strlen(job_action_name))) { + if (!val_name.empty() && 0 == strncasecmp(val_name.ptr(), job_action_name, strlen(job_action_name))) { if (OB_FAIL(dml.add_column("job_action", ObHexEscapeSqlStr(val_name)))) { LOG_WARN("failed to add column", K(ret)); } else if (OB_FAIL(dml.add_column("what", ObHexEscapeSqlStr(val_name)))) { diff --git a/src/share/stat/ob_dbms_stats_preferences.cpp b/src/share/stat/ob_dbms_stats_preferences.cpp index cea3672617..36e08a5ec1 100644 --- a/src/share/stat/ob_dbms_stats_preferences.cpp +++ b/src/share/stat/ob_dbms_stats_preferences.cpp @@ -101,9 +101,7 @@ int ObDbmsStatsPreferences::get_prefs(ObExecContext &ctx, } else if (got_result) { /*do nothing*/ } else { - ret = OB_ERR_DBMS_STATS_PL; - LOG_WARN("Invalid input values for pname", K(ret), K(opt_name)); - LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL, "Invalid input values for pname"); + result.set_null(); } LOG_TRACE("Succeed to get prefs", K(ret), K(get_user_sql), K(get_global_sql), K(result)); } @@ -544,6 +542,22 @@ int ObDbmsStatsPreferences::gen_init_global_prefs_sql(ObSqlString &raw_sql, } if (OB_SUCC(ret)) {//init estimate_block ObEstimateBlockPrefs prefs; + if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), + K(prefs.get_stat_pref_default_value())); + } else if (OB_FAIL(value_str.append_fmt("('%s', %s, %s, '%s'),", + prefs.get_stat_pref_name(), + null_str, + time_str, + prefs.get_stat_pref_default_value()))) { + LOG_WARN("failed to append", K(ret)); + } else { + ++ total_rows; + } + } + if (OB_SUCC(ret)) {//init block_sample + ObBlockSamplePrefs prefs; if (OB_ISNULL(prefs.get_stat_pref_name()) || OB_ISNULL(prefs.get_stat_pref_default_value())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(prefs.get_stat_pref_name()), @@ -1071,6 +1085,25 @@ int ObEstimateBlockPrefs::check_pref_value_validity(ObTableStatParam *param/*def return ret; } +int ObBlockSamplePrefs::check_pref_value_validity(ObTableStatParam *param/*default null*/) +{ + int ret = OB_SUCCESS; + if (pvalue_.empty() || 0 == pvalue_.case_compare("FALSE")) { + if (param != NULL) { + param->sample_info_.set_is_block_sample(false); + } + } else if (0 == pvalue_.case_compare("TRUE")) { + if (param != NULL) { + param->sample_info_.set_is_block_sample(true); + } + } else { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("Illegal value for BLOCK_SAMPLE", K(ret), K(pvalue_)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL,"Illegal value for BLOCK_SAMPLE: must be {TRUE, FALSE}"); + } + return ret; +} + #define ISSPACE(c) ((c) == ' ' || (c) == '\n' || (c) == '\r' || (c) == '\t' || (c) == '\f' || (c) == '\v') //compatible oracle, global prefs/schema prefs just only can set "for all columns...." diff --git a/src/share/stat/ob_dbms_stats_preferences.h b/src/share/stat/ob_dbms_stats_preferences.h index 9226e25d52..751931c2a5 100644 --- a/src/share/stat/ob_dbms_stats_preferences.h +++ b/src/share/stat/ob_dbms_stats_preferences.h @@ -214,6 +214,19 @@ class ObEstimateBlockPrefs : public ObStatPrefs virtual const char* get_stat_pref_default_value() const { return "TRUE"; } }; +class ObBlockSamplePrefs : public ObStatPrefs +{ + public: + ObBlockSamplePrefs() : ObStatPrefs() {} + ObBlockSamplePrefs(ObIAllocator *alloc, + ObSQLSessionInfo *session_info, + const ObString &pvalue) : + ObStatPrefs(alloc, session_info, pvalue) {} + virtual int check_pref_value_validity(ObTableStatParam *param = NULL) override; + virtual const char* get_stat_pref_name() const { return "BLOCK_SAMPLE"; } + virtual const char* get_stat_pref_default_value() const { return "FALSE"; } +}; + template static int new_stat_prefs(ObIAllocator &allocator, ObSQLSessionInfo *session_info, const ObString &opt_value, T *&src) diff --git a/src/share/stat/ob_dbms_stats_utils.cpp b/src/share/stat/ob_dbms_stats_utils.cpp index 342aef7007..057c41ca05 100644 --- a/src/share/stat/ob_dbms_stats_utils.cpp +++ b/src/share/stat/ob_dbms_stats_utils.cpp @@ -23,6 +23,7 @@ #include "share/stat/ob_stat_item.h" #include "share/schema/ob_part_mgr_util.h" #include "sql/engine/expr/ob_expr_lob_utils.h" +#include "sql/ob_result_set.h" #ifdef OB_BUILD_ORACLE_PL #include "pl/sys_package/ob_json_pl_utils.h" @@ -660,7 +661,10 @@ int ObDbmsStatsUtils::merge_col_stats(const ObTableStatParam ¶m, LOG_WARN("get unexpected null pointer", K(ret)); } else if (is_part_id_valid(param, col_stat->get_partition_id())) { col_stat->set_num_distinct(ObGlobalNdvEval::get_ndv_from_llc(col_stat->get_llc_bitmap())); - if (OB_FAIL(dst_col_stats.push_back(col_stat))) { + if (OB_UNLIKELY(col_stat->get_num_distinct() < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", KPC(col_stat), K(old_col_stats), K(ret)); + } else if (OB_FAIL(dst_col_stats.push_back(col_stat))) { LOG_WARN("fail to push back table stats", K(ret)); } } @@ -1074,8 +1078,10 @@ int ObDbmsStatsUtils::prepare_gather_stat_param(const ObTableStatParam ¶m, gather_param.stat_level_ = stat_level; if (stat_level == SUBPARTITION_LEVEL) { gather_param.need_histogram_ = param.subpart_stat_param_.gather_histogram_; + gather_param.is_specify_partition_ = param.subpart_infos_.count() != param.all_subpart_infos_.count(); } else if (stat_level == PARTITION_LEVEL) { gather_param.need_histogram_ = param.part_stat_param_.gather_histogram_; + gather_param.is_specify_partition_ = param.part_infos_.count() != param.all_part_infos_.count(); } else if (stat_level == TABLE_LEVEL) { gather_param.need_histogram_ = param.global_stat_param_.gather_histogram_; } @@ -1274,6 +1280,24 @@ int ObDbmsStatsUtils::check_all_cols_range_skew(const ObIArrayget_optimizer_features_enable_version(optimizer_features_enable_version))) { + LOG_WARN("failed to get_optimizer_features_enable_version", K(ret)); + } else if (optimizer_features_enable_version < COMPAT_VERSION_4_2_4 || + (optimizer_features_enable_version >= COMPAT_VERSION_4_3_0 && + optimizer_features_enable_version < COMPAT_VERSION_4_3_2)) { + //do nothing + } else if (OB_FAIL(ObResultSet::implicit_commit_before_cmd_execute(*ctx.get_my_session(), ctx, stmt::T_ANALYZE))) { + LOG_WARN("failed to implicit commit before cmd execute", K(ret)); + } else {/*do nothing*/} + return ret; +} } } diff --git a/src/share/stat/ob_dbms_stats_utils.h b/src/share/stat/ob_dbms_stats_utils.h index c107d917cf..2dfa9dbe63 100644 --- a/src/share/stat/ob_dbms_stats_utils.h +++ b/src/share/stat/ob_dbms_stats_utils.h @@ -181,6 +181,8 @@ public: static int check_all_cols_range_skew(const ObIArray &column_params, ObIArray &opt_stats); + static int implicit_commit_before_gather_stats(sql::ObExecContext &ctx); + private: static int batch_write(share::schema::ObSchemaGetterGuard *schema_guard, const uint64_t tenant_id, diff --git a/src/share/stat/ob_hybrid_hist_estimator.cpp b/src/share/stat/ob_hybrid_hist_estimator.cpp index b253e44d62..e2e81ee9d8 100644 --- a/src/share/stat/ob_hybrid_hist_estimator.cpp +++ b/src/share/stat/ob_hybrid_hist_estimator.cpp @@ -354,16 +354,28 @@ int ObHybridHistEstimator::compute_estimate_percent(int64_t total_row_count, } if (OB_SUCC(ret) && need_sample) { if (total_row_count * est_percent / 100 >= MAGIC_MIN_SAMPLE_SIZE) { - /*do nothing*/ + const int64_t MAGIC_MAX_SPECIFY_SAMPLE_SIZE = 1000000; + is_block_sample = !is_block_sample ? total_row_count >= MAX_AUTO_GATHER_FULL_TABLE_ROWS : is_block_sample; + int64_t max_allowed_multiple = max_num_bkts <= ObColumnStatParam::DEFAULT_HISTOGRAM_BUCKET_NUM ? 1 : + max_num_bkts / ObColumnStatParam::DEFAULT_HISTOGRAM_BUCKET_NUM; + int64_t max_specify_sample_size = MAGIC_MAX_SPECIFY_SAMPLE_SIZE * max_allowed_multiple; + if (total_row_count * est_percent / 100 >= max_specify_sample_size) { + est_percent = max_specify_sample_size * 100.0 / total_row_count; + } } else if (total_row_count <= MAGIC_SAMPLE_SIZE) { need_sample = false; est_percent = 0.0; is_block_sample = false; } else { - is_block_sample = false; + is_block_sample = total_row_count >= MAX_AUTO_GATHER_FULL_TABLE_ROWS; est_percent = (MAGIC_SAMPLE_SIZE * 100.0) / total_row_count; } } + } else if (total_row_count >= MAX_AUTO_GATHER_FULL_TABLE_ROWS) { + need_sample = true; + is_block_sample = true; + const int64_t MAGIC_MAX_SAMPLE_SIZE = 100000; + est_percent = MAGIC_MAX_SAMPLE_SIZE * 100.0 / total_row_count; } else if (total_row_count >= MAGIC_MAX_AUTO_SAMPLE_SIZE) { if (max_num_bkts <= ObColumnStatParam::DEFAULT_HISTOGRAM_BUCKET_NUM) { need_sample = true; diff --git a/src/share/stat/ob_incremental_stat_estimator.cpp b/src/share/stat/ob_incremental_stat_estimator.cpp index f5d8ca1127..54939a9a13 100644 --- a/src/share/stat/ob_incremental_stat_estimator.cpp +++ b/src/share/stat/ob_incremental_stat_estimator.cpp @@ -227,6 +227,8 @@ int ObIncrementalStatEstimator::derive_split_gather_stats(ObExecContext &ctx, } else if (OB_ISNULL(param.allocator_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(param)); + } else if (OB_FAIL(THIS_WORKER.check_status())) { + LOG_WARN("check status failed", KR(ret)); } else { ObArenaAllocator allocator("IncrementStats", OB_MALLOC_NORMAL_BLOCK_SIZE, param.tenant_id_); ObSEArray cur_table_stats; diff --git a/src/share/stat/ob_index_stats_estimator.cpp b/src/share/stat/ob_index_stats_estimator.cpp index 459384395f..a2d24537ce 100644 --- a/src/share/stat/ob_index_stats_estimator.cpp +++ b/src/share/stat/ob_index_stats_estimator.cpp @@ -313,6 +313,8 @@ int ObIndexStatsEstimator::fast_gather_index_stats(ObExecContext &ctx, } else { index_stat->set_macro_block_num(block_num_stat->tab_macro_cnt_); index_stat->set_micro_block_num(block_num_stat->tab_micro_cnt_); + index_stat->set_sstable_row_count(block_num_stat->sstable_row_cnt_); + index_stat->set_memtable_row_count(block_num_stat->memtable_row_cnt_); } if (OB_SUCC(ret)) { if (OB_FAIL(index_table_stats.push_back(index_stat))) { diff --git a/src/share/stat/ob_opt_stat_sql_service.cpp b/src/share/stat/ob_opt_stat_sql_service.cpp index 9c563075fd..3915f930cc 100644 --- a/src/share/stat/ob_opt_stat_sql_service.cpp +++ b/src/share/stat/ob_opt_stat_sql_service.cpp @@ -908,11 +908,11 @@ int ObOptStatSqlService::get_table_stat_sql(const uint64_t tenant_id, OB_FAIL(dml_splicer.add_column("object_type", stat.get_object_type())) || OB_FAIL(dml_splicer.add_time_column("last_analyzed", stat.get_last_analyzed() == 0 ? current_time : stat.get_last_analyzed())) || - OB_FAIL(dml_splicer.add_column("sstable_row_count", -1)) || + OB_FAIL(dml_splicer.add_column("sstable_row_count", stat.get_sstable_row_count())) || OB_FAIL(dml_splicer.add_column("sstable_avg_row_len", -1)) || OB_FAIL(dml_splicer.add_column("macro_blk_cnt", stat.get_macro_block_num())) || OB_FAIL(dml_splicer.add_column("micro_blk_cnt", stat.get_micro_block_num())) || - OB_FAIL(dml_splicer.add_column("memtable_row_cnt", -1)) || + OB_FAIL(dml_splicer.add_column("memtable_row_cnt", stat.get_memtable_row_count())) || OB_FAIL(dml_splicer.add_column("memtable_avg_row_len", -1)) || OB_FAIL(dml_splicer.add_column("row_cnt", stat.get_row_count())) || OB_FAIL(dml_splicer.add_column("avg_row_len", stat.get_avg_row_size())) || @@ -951,8 +951,9 @@ int ObOptStatSqlService::get_column_stat_sql(const uint64_t tenant_id, uint64_t data_version = 0; if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { LOG_WARN("fail to get tenant data version", KR(ret), K(tenant_id), K(data_version)); - } else if (OB_UNLIKELY(ObHistType::INVALID_TYPE != stat.get_histogram().get_type() && - stat.get_histogram().get_bucket_cnt() == 0)) { + } else if (OB_UNLIKELY((ObHistType::INVALID_TYPE != stat.get_histogram().get_type() && + stat.get_histogram().get_bucket_cnt() == 0) || + stat.get_num_distinct() < 0)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(stat)); } else if (OB_FAIL(get_valid_obj_str(stat.get_min_value(), min_meta, allocator, min_str, print_params)) || @@ -1001,7 +1002,7 @@ int ObOptStatSqlService::get_column_stat_sql(const uint64_t tenant_id, OB_FAIL(dml_splicer.add_column("distinct_cnt_synopsis", llc_hex_buf == NULL ? "" : llc_hex_buf)) || OB_FAIL(dml_splicer.add_column("distinct_cnt_synopsis_size", llc_comp_size * 2)) || OB_FAIL(dml_splicer.add_column("sample_size", stat.get_histogram().get_sample_size())) || - OB_FAIL(dml_splicer.add_column("density", stat.get_histogram().get_density())) || + OB_FAIL(dml_splicer.add_long_double_column("density", stat.get_histogram().get_density())) || OB_FAIL(dml_splicer.add_column("bucket_cnt", stat.get_histogram().get_bucket_cnt())) || OB_FAIL(dml_splicer.add_column("histogram_type", stat.get_histogram().get_type())) || OB_FAIL(dml_splicer.add_column("global_stats", 0)) || diff --git a/src/share/stat/ob_stat_define.cpp b/src/share/stat/ob_stat_define.cpp index ded92ea5c9..abcabb477d 100644 --- a/src/share/stat/ob_stat_define.cpp +++ b/src/share/stat/ob_stat_define.cpp @@ -284,5 +284,27 @@ bool ObTableStatParam::is_specify_column_gather() const return is_specify; } +int64_t ObTableStatParam::get_need_gather_column() const +{ + int64_t valid_column = 0; + for (int64_t i = 0; i < column_params_.count(); ++i) { + if (column_params_.at(i).need_basic_stat()) { + ++ valid_column; + } + } + return valid_column; +} + +int64_t ObOptStatGatherParam::get_need_gather_column() const +{ + int64_t valid_column = 0; + for (int64_t i = 0; i < column_params_.count(); ++i) { + if (column_params_.at(i).need_basic_stat()) { + ++ valid_column; + } + } + return valid_column; +} + } } diff --git a/src/share/stat/ob_stat_define.h b/src/share/stat/ob_stat_define.h index c42415f743..8695be95fc 100644 --- a/src/share/stat/ob_stat_define.h +++ b/src/share/stat/ob_stat_define.h @@ -56,7 +56,7 @@ enum StatOptionFlags const static double OPT_DEFAULT_STALE_PERCENT = 0.1; const static int64_t OPT_DEFAULT_STATS_RETENTION = 31; const static int64_t OPT_STATS_MAX_VALUE_CHAR_LEN = 128; -const static int64_t OPT_STATS_BIG_TABLE_ROWS = 10000000; +const int64_t MAX_AUTO_GATHER_FULL_TABLE_ROWS = 100000000; const int64_t MAGIC_SAMPLE_SIZE = 5500; const int64_t MAGIC_MAX_AUTO_SAMPLE_SIZE = 22000; const int64_t MAGIC_MIN_SAMPLE_SIZE = 2500; @@ -138,7 +138,9 @@ struct BlockNumStat tab_macro_cnt_(0), tab_micro_cnt_(0), cg_macro_cnt_arr_(), - cg_micro_cnt_arr_() + cg_micro_cnt_arr_(), + sstable_row_cnt_(0), + memtable_row_cnt_(0) { cg_macro_cnt_arr_.set_attr(ObMemAttr(MTL_ID(), "BlockNumStat")); cg_micro_cnt_arr_.set_attr(ObMemAttr(MTL_ID(), "BlockNumStat")); @@ -147,10 +149,14 @@ struct BlockNumStat int64_t tab_micro_cnt_; ObSEArray cg_macro_cnt_arr_; ObSEArray cg_micro_cnt_arr_; + int64_t sstable_row_cnt_; + int64_t memtable_row_cnt_; TO_STRING_KV(K(tab_macro_cnt_), K(tab_micro_cnt_), K(cg_macro_cnt_arr_), - K(cg_micro_cnt_arr_)) + K(cg_micro_cnt_arr_), + K(sstable_row_cnt_), + K(memtable_row_cnt_)) }; //TODO@jiangxiu.wt: improve the expression of PartInfo, use the map is better. @@ -475,6 +481,12 @@ struct ObTableStatParam { bool is_specify_column_gather() const; + int64_t get_need_gather_column() const; + + bool need_gather_stats() const { return global_stat_param_.need_modify_ || + part_stat_param_.need_modify_ || + subpart_stat_param_.need_modify_; } + uint64_t tenant_id_; ObString db_name_; @@ -599,9 +611,11 @@ struct ObOptStatGatherParam { global_part_id_(-1), gather_vectorize_(DEFAULT_STAT_GATHER_VECTOR_BATCH_SIZE), sepcify_scn_(0), - use_column_store_(false) + use_column_store_(false), + is_specify_partition_(false) {} int assign(const ObOptStatGatherParam &other); + int64_t get_need_gather_column() const; uint64_t tenant_id_; ObString db_name_; ObString tab_name_; @@ -625,6 +639,7 @@ struct ObOptStatGatherParam { int64_t gather_vectorize_; uint64_t sepcify_scn_; bool use_column_store_; + bool is_specify_partition_; TO_STRING_KV(K(tenant_id_), K(db_name_), @@ -646,7 +661,8 @@ struct ObOptStatGatherParam { K(global_part_id_), K(gather_vectorize_), K(sepcify_scn_), - K(use_column_store_)); + K(use_column_store_), + K(is_specify_partition_)); }; struct ObOptStat @@ -664,9 +680,20 @@ struct ObOptStat struct ObHistogramParam { + ObHistogramParam(): + epc_(0), + minval_(NULL), + maxval_(NULL), + bkvals_(), + novals_(), + chvals_(), + eavals_(), + rpcnts_(), + eavs_(0) + {} int64_t epc_; //Number of buckets in histogram - ObString minval_; //Minimum value - ObString maxval_; //Maximum value + const ObObj *minval_; //Minimum value + const ObObj *maxval_; //Maximum value ObSEArray bkvals_; //Array of bucket numbers ObSEArray novals_; //Array of normalized end point values ObSEArray chvals_; //Array of dumped end point values @@ -712,14 +739,24 @@ struct ObSetTableStatParam struct ObSetColumnStatParam { + ObSetColumnStatParam(): + table_param_(), + distcnt_(0), + density_(0.0), + nullcnt_(0), + hist_param_(), + avgclen_(0), + flags_(0), + col_meta_() + {} ObTableStatParam table_param_; - int64_t distcnt_; double density_; int64_t nullcnt_; ObHistogramParam hist_param_; int64_t avgclen_; int64_t flags_; + common::ObObjMeta col_meta_; TO_STRING_KV(K(table_param_), K(distcnt_), @@ -727,7 +764,8 @@ struct ObSetColumnStatParam K(nullcnt_), K(hist_param_), K(avgclen_), - K(flags_)); + K(flags_), + K(col_meta_)); }; diff --git a/src/share/stat/ob_stat_item.cpp b/src/share/stat/ob_stat_item.cpp index 70feea23ed..59d4a98321 100644 --- a/src/share/stat/ob_stat_item.cpp +++ b/src/share/stat/ob_stat_item.cpp @@ -509,7 +509,8 @@ void ObGlobalTableStat::add(int64_t rc, int64_t rs, int64_t ds, int64_t mac, int } int ObGlobalTableStat::add(int64_t rc, int64_t rs, int64_t ds, int64_t mac, int64_t mic, - ObIArray &cg_macro_arr, ObIArray &cg_micro_arr) + ObIArray &cg_macro_arr, ObIArray &cg_micro_arr, + int64_t scnt, int64_t mcnt) { // skip empty partition int ret = OB_SUCCESS; @@ -520,6 +521,8 @@ int ObGlobalTableStat::add(int64_t rc, int64_t rs, int64_t ds, int64_t mac, int6 macro_block_count_ += mac; micro_block_count_ += mic; part_cnt_ ++; + sstable_row_cnt_ += scnt; + memtable_row_cnt_ += mcnt; if (cg_macro_arr.empty()) { //do nothing } else if (cg_macro_cnt_arr_.empty()) { diff --git a/src/share/stat/ob_stat_item.h b/src/share/stat/ob_stat_item.h index 37a4cbb853..17398afe18 100644 --- a/src/share/stat/ob_stat_item.h +++ b/src/share/stat/ob_stat_item.h @@ -301,13 +301,14 @@ public: ObGlobalTableStat() : row_count_(0), row_size_(0), data_size_(0), macro_block_count_(0), micro_block_count_(0), part_cnt_(0), last_analyzed_(0), - cg_macro_cnt_arr_(), cg_micro_cnt_arr_(), - stat_locked_(false) + cg_macro_cnt_arr_(), cg_micro_cnt_arr_(), stat_locked_(false), + sstable_row_cnt_(0), memtable_row_cnt_(0) {} void add(int64_t rc, int64_t rs, int64_t ds, int64_t mac, int64_t mic); int add(int64_t rc, int64_t rs, int64_t ds, int64_t mac, int64_t mic, - ObIArray &cg_macro_arr, ObIArray &cg_micro_arr); + ObIArray &cg_macro_arr, ObIArray &cg_micro_arr, + int64_t scnt, int64_t mcnt); int64_t get_row_count() const; int64_t get_avg_row_size() const; @@ -320,6 +321,8 @@ public: void set_last_analyzed(int64_t last_analyzed) { last_analyzed_ = last_analyzed; } void set_stat_locked(bool locked) { stat_locked_ = locked; } bool get_stat_locked() const { return stat_locked_; } + int64_t get_sstable_row_cnt() const { return sstable_row_cnt_; } + int64_t get_memtable_row_cnt() const { return memtable_row_cnt_; } TO_STRING_KV(K(row_count_), @@ -331,7 +334,9 @@ public: K(last_analyzed_), K(cg_macro_cnt_arr_), K(cg_micro_cnt_arr_), - K(stat_locked_)); + K(stat_locked_), + K(sstable_row_cnt_), + K(memtable_row_cnt_)); private: int64_t row_count_; @@ -344,6 +349,8 @@ private: ObArray cg_macro_cnt_arr_; ObArray cg_micro_cnt_arr_; bool stat_locked_; + int64_t sstable_row_cnt_; + int64_t memtable_row_cnt_; }; class ObGlobalNullEval diff --git a/src/share/stat/ob_stats_estimator.cpp b/src/share/stat/ob_stats_estimator.cpp index 29b850ce1c..f1ee879a73 100644 --- a/src/share/stat/ob_stats_estimator.cpp +++ b/src/share/stat/ob_stats_estimator.cpp @@ -95,7 +95,7 @@ int ObStatsEstimator::fill_sample_info(common::ObIAllocator &alloc, bool block_sample) { int ret = OB_SUCCESS; - if (est_percent>= 0.000001 && est_percent <= 100.0) { + if (est_percent>= 0.000001 && est_percent < 100.0) { char *buf = NULL; int32_t buf_len = 50;//double类型一般15～16位，加上字符长度16左右，因此数组长度为50足够用 int64_t real_len = -1; @@ -407,6 +407,8 @@ int ObStatsEstimator::do_estimate(uint64_t tenant_id, static_cast(sql_proxy->get_pool()); sqlclient::ObISQLConnection *conn = NULL; session->set_inner_session(); + // + session->set_autocommit(true); SMART_VAR(ObMySQLProxy::MySQLResult, proxy_result) { sqlclient::ObMySQLResult *client_result = NULL; if (OB_FAIL(pool->acquire(session, conn, lib::is_oracle_mode()))) { diff --git a/src/share/system_variable/default_system_variable.json b/src/share/system_variable/default_system_variable.json index cd7ac2d612..de8551db02 100644 --- a/src/share/system_variable/default_system_variable.json +++ b/src/share/system_variable/default_system_variable.json @@ -1,7 +1,7 @@ [ { "scenario": "express_oltp", - "comment" : "for workloads like trade, payment core system, internet high throughput application, etc. no restrictions like foreign key, no stored procedure, no long transaction, no large transaction, no complex join, no complex subquery", + "comment": "for workloads like trade, payment core system, internet high throughput application, etc. no restrictions like foreign key, no stored procedure, no long transaction, no large transaction, no complex join, no complex subquery", "variables": { "tenant": [ ] @@ -9,7 +9,7 @@ }, { "scenario": "complex_oltp", - "comment" : "for workloads like bank, insurance system. they often have complex join, complex correlated subquery, batch jobs written in PL, have both long and large transactions. Sometimes use parallel execution for short running queries", + "comment": "for workloads like bank, insurance system. they often have complex join, complex correlated subquery, batch jobs written in PL, have both long and large transactions. Sometimes use parallel execution for short running queries", "variables": { "tenant": [ ] @@ -39,6 +39,21 @@ "name": "ob_sql_work_area_percentage", "value": 30, "comment":"larger sql work area can save spill cost" + }, + { + "name": "parallel_degree_policy", + "value": "AUTO", + "comment": "auto dop is enabled by default for OLAP" + }, + { + "name": "collation_server", + "value": "utf8mb4_bin", + "comment": "use binary collation can achieve 20% performance gain compared with other collations" + }, + { + "name": "collation_connection", + "value": "utf8mb4_bin", + "comment": "use binary collation can achieve 20% performance gain compared with other collations" } ] } diff --git a/src/share/system_variable/ob_system_variable_init.json b/src/share/system_variable/ob_system_variable_init.json index d6f73261b1..a4117ac860 100644 --- a/src/share/system_variable/ob_system_variable_init.json +++ b/src/share/system_variable/ob_system_variable_init.json @@ -6961,5 +6961,19 @@ "background_cn": "", "ref_url": "" + }, + "event_scheduler": { + "id": 10392, + "name": "event_scheduler", + "default_value": "0", + "base_value": "0", + "data_type": "bool", + "info": "Whether the MySQL Event Scheduler is enabled.", + "flags": "GLOBAL | MYSQL_ONLY", + "publish_version": "424", + "info_cn": "", + "background_cn": "", + "ref_url": "", + "placeholder": true } } diff --git a/src/share/table/ob_table.h b/src/share/table/ob_table.h index 3c3804d48a..162cbc4286 100644 --- a/src/share/table/ob_table.h +++ b/src/share/table/ob_table.h @@ -230,6 +230,7 @@ void ObTableEntityFactory::free_all() enum class ObQueryOperationType : int { QUERY_START = 0, QUERY_NEXT = 1, + QUERY_END = 2, QUERY_MAX }; diff --git a/src/share/vector/vector_basic_op.h b/src/share/vector/vector_basic_op.h index 2835307031..56b753c1f7 100644 --- a/src/share/vector/vector_basic_op.h +++ b/src/share/vector/vector_basic_op.h @@ -632,7 +632,17 @@ struct VecTCCmpCalc const double l = *reinterpret_cast(l_v); const double r = *reinterpret_cast(r_v); const double P = 5 / LOG_10[l_meta.get_scale() + 1]; - if (l == r || fabs(l - r) < P) { + if (isnan(l) || isnan(r)) { + if (isnan(l) && isnan(r)) { + cmp_ret = 0; + } else if (isnan(l)) { + // l is nan, r is not nan:left always bigger than right + cmp_ret = 1; + } else { + // l is not nan, r is nan, left always less than right + cmp_ret = -1; + } + } else if (l == r || fabs(l - r) < P) { cmp_ret = 0; } else { cmp_ret = (l < r ? -1: 1); diff --git a/src/sql/das/ob_das_scan_op.cpp b/src/sql/das/ob_das_scan_op.cpp index c1b3f75272..926c846338 100644 --- a/src/sql/das/ob_das_scan_op.cpp +++ b/src/sql/das/ob_das_scan_op.cpp @@ -1355,12 +1355,31 @@ int ObLocalIndexLookupOp::get_next_row(ObNewRow *&row) return OB_NOT_IMPLEMENT; } +void ObLocalIndexLookupOp::print_trans_info_and_key_range_() +{ + int ret = OB_ERR_DEFENSIVE_CHECK; + if (trans_info_array_.count() == scan_param_.key_ranges_.count()) { + for (int64_t i = 0; i < trans_info_array_.count(); i++) { + LOG_ERROR("dump TableLookup DAS Task trans_info and key_ranges", K(i), + KPC(trans_info_array_.at(i)), K(scan_param_.key_ranges_.at(i))); + } + } else { + for (int64_t i = 0; i < scan_param_.key_ranges_.count(); i++) { + LOG_ERROR("dump TableLookup DAS Task key_ranges", + K(i), K(scan_param_.key_ranges_.at(i))); + } + } +} + int ObLocalIndexLookupOp::get_next_row() { int ret = OB_SUCCESS; if (OB_FAIL(ObIndexLookupOpImpl::get_next_row())) { if (OB_ITER_END != ret) { LOG_WARN("get next row from index table lookup failed", K(ret)); + if (OB_ERR_DEFENSIVE_CHECK == ret) { + (void)print_trans_info_and_key_range_(); + } } else { LOG_DEBUG("get next row from index table lookup", K(ret)); } @@ -1374,6 +1393,9 @@ int ObLocalIndexLookupOp::get_next_rows(int64_t &count, int64_t capacity) if (OB_FAIL(ObIndexLookupOpImpl::get_next_rows(count, capacity))) { if (OB_ITER_END != ret) { LOG_WARN("get next rows from index table lookup failed", K(ret)); + if (OB_ERR_DEFENSIVE_CHECK == ret) { + (void)print_trans_info_and_key_range_(); + } } else { LOG_DEBUG("get next rows from index table lookup ", K(ret)); } @@ -1569,17 +1591,7 @@ int ObLocalIndexLookupOp::check_lookup_row_cnt() KPC_(snapshot), KPC_(tx_desc)); concurrency_control::ObDataValidationService::set_delay_resource_recycle(ls_id_); - if (trans_info_array_.count() == scan_param_.key_ranges_.count()) { - for (int64_t i = 0; i < trans_info_array_.count(); i++) { - LOG_ERROR("dump TableLookup DAS Task trans_info and key_ranges", K(i), - KPC(trans_info_array_.at(i)), K(scan_param_.key_ranges_.at(i))); - } - } else { - for (int64_t i = 0; i < scan_param_.key_ranges_.count(); i++) { - LOG_ERROR("dump TableLookup DAS Task key_ranges", - K(i), K(scan_param_.key_ranges_.at(i))); - } - } + (void)print_trans_info_and_key_range_(); } } diff --git a/src/sql/das/ob_das_scan_op.h b/src/sql/das/ob_das_scan_op.h index cd00aa748b..2e34095d5c 100644 --- a/src/sql/das/ob_das_scan_op.h +++ b/src/sql/das/ob_das_scan_op.h @@ -383,6 +383,8 @@ public: common::ObITabletScan &get_tsc_service(); protected: virtual int init_scan_param(); +protected: + void print_trans_info_and_key_range_(); protected: const ObDASScanCtDef *lookup_ctdef_; //lookup ctdef ObDASScanRtDef *lookup_rtdef_; //lookup rtdef diff --git a/src/sql/engine/aggregate/ob_aggregate_processor.cpp b/src/sql/engine/aggregate/ob_aggregate_processor.cpp index 2cb6edb17b..6643c417af 100644 --- a/src/sql/engine/aggregate/ob_aggregate_processor.cpp +++ b/src/sql/engine/aggregate/ob_aggregate_processor.cpp @@ -636,7 +636,7 @@ void ObAggregateProcessor::HybridHistExtraResult::reuse() int ObAggregateProcessor::HybridHistExtraResult::init(const uint64_t tenant_id, const ObAggrInfo &aggr_info, ObEvalCtx &eval_ctx, const bool need_rewind, - ObIOEventObserver *io_event_observer, ObSqlWorkAreaProfile &profile, + ObIOEventObserver *io_event_observer, ObMonitorNode &op_monitor_info) { int ret = OB_SUCCESS; @@ -670,7 +670,7 @@ int ObAggregateProcessor::HybridHistExtraResult::init(const uint64_t tenant_id, ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fall to alloc buff", "size", sizeof(ObMaterialOpImpl)); } else { - new (mat_op_) ObMaterialOpImpl(op_monitor_info, profile); + new (mat_op_) ObMaterialOpImpl(op_monitor_info); if (OB_FAIL(mat_op_->init(tenant_id, &eval_ctx, &eval_ctx.exec_ctx_, @@ -893,7 +893,6 @@ ObAggregateProcessor::ObAggregateProcessor(ObEvalCtx &eval_ctx, removal_info_(), support_fast_single_row_agg_(false), op_eval_infos_(nullptr), - profile_(ObSqlWorkAreaType::HASH_WORK_AREA), op_monitor_info_(op_monitor_info), need_advance_collect_(false) { @@ -2045,7 +2044,6 @@ int ObAggregateProcessor::generate_group_row(GroupRow *&new_group_row, eval_ctx_, need_rewind, io_event_observer_, - profile_, op_monitor_info_))) { LOG_WARN("init hybrid hist extra result failed"); } @@ -2248,7 +2246,6 @@ int ObAggregateProcessor::fill_group_row(GroupRow *new_group_row, eval_ctx_, need_rewind, io_event_observer_, - profile_, op_monitor_info_))) { LOG_WARN("init hybrid hist extra result failed"); } diff --git a/src/sql/engine/aggregate/ob_aggregate_processor.h b/src/sql/engine/aggregate/ob_aggregate_processor.h index 63a8cb0f5c..f994f2ebc4 100644 --- a/src/sql/engine/aggregate/ob_aggregate_processor.h +++ b/src/sql/engine/aggregate/ob_aggregate_processor.h @@ -414,7 +414,7 @@ public: int init(const uint64_t tenant_id, const ObAggrInfo &aggr_info, ObEvalCtx &eval_ctx, const bool need_rewind, - ObIOEventObserver *io_event_observer, ObSqlWorkAreaProfile &profile, + ObIOEventObserver *io_event_observer, ObMonitorNode &op_monitor_info); int add_sort_row(const ObIArray &expr, ObEvalCtx &eval_ctx); @@ -1221,7 +1221,6 @@ private: RemovalInfo removal_info_; bool support_fast_single_row_agg_; ObIArray *op_eval_infos_; - ObSqlWorkAreaProfile profile_; ObMonitorNode &op_monitor_info_; bool need_advance_collect_; }; diff --git a/src/sql/engine/basic/ob_material_op.h b/src/sql/engine/basic/ob_material_op.h index 36bf1f64bb..52f1ef29ca 100644 --- a/src/sql/engine/basic/ob_material_op.h +++ b/src/sql/engine/basic/ob_material_op.h @@ -53,9 +53,8 @@ class ObMaterialOp : public ObOperator public: ObMaterialOp(ObExecContext &exec_ctx, const ObOpSpec &spec, ObOpInput *input) : ObOperator(exec_ctx, spec, input), - profile_(ObSqlWorkAreaType::HASH_WORK_AREA), is_first_(false), - material_impl_(op_monitor_info_, profile_) + material_impl_(op_monitor_info_) {} virtual int inner_open() override; @@ -79,7 +78,6 @@ private: private: friend class ObValues; - ObSqlWorkAreaProfile profile_; bool is_first_; ObMaterialOpImpl material_impl_; }; diff --git a/src/sql/engine/basic/ob_material_op_impl.cpp b/src/sql/engine/basic/ob_material_op_impl.cpp index 90fe6c7e9c..2d64daf67c 100644 --- a/src/sql/engine/basic/ob_material_op_impl.cpp +++ b/src/sql/engine/basic/ob_material_op_impl.cpp @@ -23,7 +23,7 @@ namespace oceanbase using namespace common; namespace sql { -ObMaterialOpImpl::ObMaterialOpImpl(ObMonitorNode &op_monitor_info, ObSqlWorkAreaProfile &profile) +ObMaterialOpImpl::ObMaterialOpImpl(ObMonitorNode &op_monitor_info) : inited_(false), got_first_row_(false), tenant_id_(OB_INVALID_ID), @@ -32,8 +32,8 @@ ObMaterialOpImpl::ObMaterialOpImpl(ObMonitorNode &op_monitor_info, ObSqlWorkArea datum_store_(ObModIds::OB_HASH_NODE_GROUP_ROWS), datum_store_it_(), eval_ctx_(nullptr), - profile_(profile), - sql_mem_processor_(profile, op_monitor_info), + profile_(ObSqlWorkAreaType::HASH_WORK_AREA), + sql_mem_processor_(profile_, op_monitor_info), input_rows_(OB_INVALID_ID), input_width_(OB_INVALID_ID), op_type_(PHY_INVALID), diff --git a/src/sql/engine/basic/ob_material_op_impl.h b/src/sql/engine/basic/ob_material_op_impl.h index 30498c54a5..0077d71227 100644 --- a/src/sql/engine/basic/ob_material_op_impl.h +++ b/src/sql/engine/basic/ob_material_op_impl.h @@ -25,7 +25,7 @@ namespace sql class ObMaterialOpImpl { public: - explicit ObMaterialOpImpl(ObMonitorNode &op_monitor_info, ObSqlWorkAreaProfile &profile); + explicit ObMaterialOpImpl(ObMonitorNode &op_monitor_info); virtual ~ObMaterialOpImpl(); void reset(); @@ -109,7 +109,7 @@ private: ObChunkDatumStore datum_store_; ObChunkDatumStore::Iterator datum_store_it_; ObEvalCtx *eval_ctx_; - ObSqlWorkAreaProfile &profile_; + ObSqlWorkAreaProfile profile_; ObSqlMemMgrProcessor sql_mem_processor_; ObIOEventObserver *io_event_observer_; int64_t input_rows_; diff --git a/src/sql/engine/cmd/ob_analyze_executor.cpp b/src/sql/engine/cmd/ob_analyze_executor.cpp index e5e599c9f4..842abf31ce 100644 --- a/src/sql/engine/cmd/ob_analyze_executor.cpp +++ b/src/sql/engine/cmd/ob_analyze_executor.cpp @@ -58,6 +58,8 @@ int ObAnalyzeExecutor::execute(ObExecContext &ctx, ObAnalyzeStmt &stmt) GCTX.is_standby_cluster()) { ret = OB_NOT_SUPPORTED; LOG_USER_ERROR(OB_NOT_SUPPORTED, "analyze table during restore or standby cluster"); + } else if (OB_FAIL(ObDbmsStatsUtils::implicit_commit_before_gather_stats(ctx))) { + LOG_WARN("failed to implicit commit before gather stats", K(ret)); } else if (OB_FAIL(stmt.fill_table_stat_params(ctx, params))) { LOG_WARN("failed to fill table stat param", K(ret)); } else { @@ -68,12 +70,27 @@ int ObAnalyzeExecutor::execute(ObExecContext &ctx, ObAnalyzeStmt &stmt) } if (OB_SUCC(ret)) { if (stmt.is_delete_histogram()) { - //must be only one param - if (OB_FAIL(ObDbmsStatsExecutor::delete_table_stats(ctx, params.at(0), true))) { - LOG_WARN("failed to drop table stats", K(ret)); + bool cascade_columns = true; + bool cascade_indexes = true; + if (OB_UNLIKELY(params.count() != 1)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(params)); } else { - LOG_TRACE("succeed to drop table stats", K(params)); + ObArenaAllocator tmp_alloc("DeleteStats", OB_MALLOC_NORMAL_BLOCK_SIZE, params.at(0).tenant_id_); + params.at(0).allocator_ = &tmp_alloc;//use the temp allocator to free memory after delete stats. + if (OB_FAIL(ObDbmsStatsLockUnlock::check_stat_locked(ctx, params.at(0)))) { + LOG_WARN("failed to check stat locked", K(ret)); + } else if (OB_FAIL(ObDbmsStatsExecutor::delete_table_stats(ctx, params.at(0), cascade_columns))) { + LOG_WARN("failed to delete table stats", K(ret)); + } else if (OB_FAIL(pl::ObDbmsStats::update_stat_cache(session->get_rpc_tenant_id(), params.at(0)))) { + LOG_WARN("failed to update stat cache", K(ret)); + } else if (cascade_indexes && params.at(0).part_name_.empty()) { + if (OB_FAIL(pl::ObDbmsStats::delete_table_index_stats(ctx, params.at(0)))) { + LOG_WARN("failed to delete index stats", K(ret)); + } else {/*do nothing*/} + } } + LOG_TRACE("succeed to drop table stats", K(params)); } else { int64_t task_cnt = params.count(); int64_t start_time = ObTimeUtility::current_time(); diff --git a/src/sql/engine/cmd/ob_table_executor.cpp b/src/sql/engine/cmd/ob_table_executor.cpp index 1a5468c138..6eb6242957 100644 --- a/src/sql/engine/cmd/ob_table_executor.cpp +++ b/src/sql/engine/cmd/ob_table_executor.cpp @@ -109,8 +109,9 @@ int ObCreateTableExecutor::ObInsSQLPrinter::inner_print(char *buf, int64_t buf_l LOG_WARN("null stmt", K(ret)); } else if (OB_FAIL(databuff_printf(buf, buf_len, pos1, do_osg_ - ? "insert /*+GATHER_OPTIMIZER_STATISTICS*/ into %c%.*s%c.%c%.*s%c" - : "insert /*+NO_GATHER_OPTIMIZER_STATISTICS*/ into %c%.*s%c.%c%.*s%c", + ? "insert /*+ ENABLE_PARALLEL_DML PARALLEL(%lu) GATHER_OPTIMIZER_STATISTICS*/ into %c%.*s%c.%c%.*s%c" + : "insert /*+ ENABLE_PARALLEL_DML PARALLEL(%lu) NO_GATHER_OPTIMIZER_STATISTICS*/ into %c%.*s%c.%c%.*s%c", + stmt_->get_parallelism(), sep_char, stmt_->get_database_name().length(), stmt_->get_database_name().ptr(), diff --git a/src/sql/engine/expr/ob_expr.cpp b/src/sql/engine/expr/ob_expr.cpp index 0e6b928af1..06cc453a9c 100644 --- a/src/sql/engine/expr/ob_expr.cpp +++ b/src/sql/engine/expr/ob_expr.cpp @@ -1153,7 +1153,7 @@ int ObExpr::eval_vector(ObEvalCtx &ctx, ret = init_vector(ctx, batch_result_ ? VEC_UNIFORM : VEC_UNIFORM_CONST, BATCH_SIZE()); } if (OB_FAIL(ret)) { - } else if (info.projected_ || NULL == eval_batch_func_ + } else if ((batch_result_ && info.projected_) || NULL == eval_batch_func_ || (!batch_result_ && info.evaluated_)) { // expr values is projected by child or has no evaluate func, do nothing. } else if (!info.evaluated_) { diff --git a/src/sql/engine/expr/ob_expr_eval_functions.cpp b/src/sql/engine/expr/ob_expr_eval_functions.cpp index 05f9608512..8b45feb1f5 100644 --- a/src/sql/engine/expr/ob_expr_eval_functions.cpp +++ b/src/sql/engine/expr/ob_expr_eval_functions.cpp @@ -1193,6 +1193,10 @@ static ObExpr::EvalFunc g_expr_eval_functions[] = { NULL, // ObExprRbOrNull2empty::eval_rb_or_null2empty, /* 714 */ NULL, // ObExprRbAndnotNull2empty::eval_rb_andnot_null2empty, /* 715 */ NULL, //ObExprSdoRelate::eval_sdo_relate /* 716 */ + NULL, // ObExprRbToString::eval_rb_to_string, /* 717 */ + NULL, // ObExprRbFromString::eval_rb_from_string, /* 718 */ + NULL, // ObExprRbIterate::eval_rb_iterate, /* 719 */ + NULL, // ObExprArray::eval_array, /* 720 */ }; static ObExpr::EvalBatchFunc g_expr_eval_batch_functions[] = { diff --git a/src/sql/engine/expr/ob_expr_func_round.cpp b/src/sql/engine/expr/ob_expr_func_round.cpp index 501cce07dc..1235789c64 100644 --- a/src/sql/engine/expr/ob_expr_func_round.cpp +++ b/src/sql/engine/expr/ob_expr_func_round.cpp @@ -305,7 +305,7 @@ static int do_round_by_type( } case ObDecimalIntType: { if (OB_FAIL(ObExprFuncRound::calc_round_decimalint( - in_meta, out_meta, round_scale, x_datum, res_datum))) { + in_meta, out_meta, GET_SCALE_FOR_CALC(round_scale), x_datum, res_datum))) { LOG_WARN("calc_round_decimalint failed", K(ret), K(in_meta), K(out_meta), K(round_scale)); } break; @@ -434,7 +434,7 @@ static int do_round_by_type_batch_with_check(const int64_t scale, const ObExpr & if (x_datum.is_null()) { results[i].set_null(); } else if (OB_FAIL(ObExprFuncRound::calc_round_decimalint( - in_meta, out_meta, scale, x_datum, results[i]))) { + in_meta, out_meta, GET_SCALE_FOR_CALC(scale), x_datum, results[i]))) { LOG_WARN("calc_round_decimalint failed", K(ret), K(in_meta), K(out_meta), K(scale)); } } @@ -567,7 +567,7 @@ static int do_round_by_type_vector(const int64_t scale, const ObExpr &expr, for (int64_t j = bound.start(); OB_SUCC(ret) && j < bound.end(); ++j) { CHECK_ROUND_VECTOR(); if (OB_FAIL((ObExprFuncRound::calc_round_decimalint)( - in_meta, out_meta, scale, left_vec, res_vec, j))) { + in_meta, out_meta, GET_SCALE_FOR_CALC(scale), left_vec, res_vec, j))) { LOG_WARN("calc_round_decimalint failed", K(ret), K(in_meta), K(out_meta), K(scale)); } eval_flags.set(j); @@ -656,7 +656,7 @@ static int do_round_by_type_batch_without_check(const int64_t scale, const ObExp const ObDatumMeta &out_meta = expr.datum_meta_; for (int64_t i = 0; OB_SUCC(ret) && i < batch_size; ++i) { if (OB_FAIL(ObExprFuncRound::calc_round_decimalint( - in_meta, out_meta, scale, x_datums[i], results[i]))) { + in_meta, out_meta, GET_SCALE_FOR_CALC(scale), x_datums[i], results[i]))) { LOG_WARN("calc_round_decimalint failed", K(ret), K(in_meta), K(out_meta), K(scale)); } } diff --git a/src/sql/engine/expr/ob_expr_not.cpp b/src/sql/engine/expr/ob_expr_not.cpp index 4908274ae4..046a300fc0 100644 --- a/src/sql/engine/expr/ob_expr_not.cpp +++ b/src/sql/engine/expr/ob_expr_not.cpp @@ -43,7 +43,13 @@ int ObExprNot::calc_result_type1(ObExprResType &type, if (ObMaxType == type1.get_type()) { ret = OB_ERR_UNEXPECTED; } else { - type.set_int(); + if ((GET_MIN_CLUSTER_VERSION() >= MOCK_CLUSTER_VERSION_4_2_1_7 && GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_2_2_0) || + (GET_MIN_CLUSTER_VERSION() >= MOCK_CLUSTER_VERSION_4_2_4_0 && GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_3_0_0) || + GET_MIN_CLUSTER_VERSION() >= CLUSTER_VERSION_4_3_2_0) { + type.set_int32(); + } else { + type.set_int(); + } ObExprOperator::calc_result_flag1(type, type1); type.set_scale(DEFAULT_SCALE_FOR_INTEGER); type.set_precision(DEFAULT_PRECISION_FOR_BOOL); diff --git a/src/sql/engine/expr/ob_expr_util.cpp b/src/sql/engine/expr/ob_expr_util.cpp index 7b06d87bbe..9e4ddeef27 100644 --- a/src/sql/engine/expr/ob_expr_util.cpp +++ b/src/sql/engine/expr/ob_expr_util.cpp @@ -459,13 +459,13 @@ int ObExprUtil::get_mb_str_info(const ObString &str, double ObExprUtil::round_double(double val, int64_t dec) { - const double pow_val = std::pow(10, static_cast(std::abs(dec))); - volatile double val_div_tmp = val / pow_val; - volatile double val_mul_tmp = val * pow_val; - volatile double res = 0.0; + const double pow_val = std::pow(10.0, static_cast(std::abs(dec))); + double val_div_tmp = val / pow_val; + double val_mul_tmp = val * pow_val; + double res = 0.0; if (dec < 0 && std::isinf(pow_val)) { res = 0.0; - } else if (dec >= 0 && std::isinf(val_mul_tmp)) { + } else if (dec >= 0 && !std::isfinite(val_mul_tmp)) { res = val; } else { res = dec < 0 ? rint(val_div_tmp) * pow_val : rint(val_mul_tmp) / pow_val; @@ -474,23 +474,6 @@ double ObExprUtil::round_double(double val, int64_t dec) return res; } -double ObExprUtil::round_double_nearest(double val, int64_t dec) -{ - const double pow_val = std::pow(10, static_cast(std::abs(dec))); - volatile double val_div_tmp = val / pow_val; - volatile double val_mul_tmp = val * pow_val; - volatile double res = 0.0; - if (dec < 0 && std::isinf(pow_val)) { - res = 0.0; - } else if (dec >= 0 && std::isinf(val_mul_tmp)) { - res = val; - } else { - res = dec < 0 ? std::round(val_div_tmp) * pow_val : std::round(val_mul_tmp) / pow_val; - } - LOG_DEBUG("round double done", K(val), K(dec), K(res)); - return res; -} - uint64_t ObExprUtil::round_uint64(uint64_t val, int64_t dec) { uint64_t res = 0; @@ -524,7 +507,7 @@ double ObExprUtil::trunc_double(double val, int64_t dec) volatile double res = 0.0; if (dec < 0 && std::isinf(pow_val)) { res = 0.0; - } else if (dec >= 0 && std::isinf(val_mul_tmp)) { + } else if (dec >= 0 && !std::isfinite(val_mul_tmp)) { res = val; } else { if (val >= 0) { diff --git a/src/sql/engine/expr/ob_expr_util.h b/src/sql/engine/expr/ob_expr_util.h index 9e3db848bb..e9dfa98e98 100644 --- a/src/sql/engine/expr/ob_expr_util.h +++ b/src/sql/engine/expr/ob_expr_util.h @@ -80,8 +80,6 @@ public: common::ObIArray &byte_offset); // 将double round到小数点后或者小数点前指定位置 static double round_double(double val, int64_t dec); - // 将double round到小数点后指定位置，.5 -> 1 而不是0 - static double round_double_nearest(double val, int64_t dec); static uint64_t round_uint64(uint64_t val, int64_t dec); // 将double trunc到小数点后或者小数点前指定位置 static double trunc_double(double val, int64_t dec); diff --git a/src/sql/engine/px/ob_px_data_ch_provider.cpp b/src/sql/engine/px/ob_px_data_ch_provider.cpp index 74606d3860..147b6947a3 100644 --- a/src/sql/engine/px/ob_px_data_ch_provider.cpp +++ b/src/sql/engine/px/ob_px_data_ch_provider.cpp @@ -270,7 +270,7 @@ int ObPxReceiveChProvider::get_data_ch_nonblock( ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid msg", K(child_dfo_id), K(ret)); } else if (OB_FAIL(ObPxChProviderUtil::check_status(timeout_ts, qc_addr, query_start_time))) { - // nop + LOG_WARN("Fail to check status", K(child_dfo_id), K(msg_set_), K(msgs_), K(ret)); } else { ObLockGuard lock_guard(lock_); ARRAY_FOREACH_X(msgs_, idx, cnt, OB_SUCC(ret) && !found) { @@ -329,7 +329,7 @@ int ObPxReceiveChProvider::get_data_ch( } if (OB_SUCC(ret) && !found) { ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("no receive ch found for dfo", K(child_dfo_id), K(ret)); + LOG_WARN("no receive ch found for dfo", K(child_dfo_id), K(msg_set_), K(msgs_), K(ret)); } } return ret; @@ -374,7 +374,7 @@ int ObPxReceiveChProvider::wait_msg(int64_t child_dfo_id, int64_t timeout_ts) } } if (OB_SUCC(ret)) { - while (!msg_set_[child_dfo_id]) { + while (!is_msg_set(child_dfo_id)) { msg_ready_cond_.lock(); if (!msg_set_[child_dfo_id]) { msg_ready_cond_.wait_us(1 * 1000); /* 1 ms */ @@ -392,7 +392,7 @@ int ObPxReceiveChProvider::wait_msg(int64_t child_dfo_id, int64_t timeout_ts) ObInterruptCode code = GET_INTERRUPT_CODE(); ret = code.code_; LOG_WARN("receive channel provider wait msg loop is interrupted", - K(child_dfo_id), K(wait_count), K(code), K(ret)); + K(child_dfo_id), K(wait_count), K(code), K(msg_set_[child_dfo_id]), K(ret)); break; } else { ret = OB_EAGAIN; diff --git a/src/sql/engine/px/ob_px_data_ch_provider.h b/src/sql/engine/px/ob_px_data_ch_provider.h index 89e3465af5..42c369133d 100644 --- a/src/sql/engine/px/ob_px_data_ch_provider.h +++ b/src/sql/engine/px/ob_px_data_ch_provider.h @@ -92,6 +92,11 @@ private: /* functions */ int wait_msg(int64_t child_dfo_id, int64_t timeout_ts); int reserve_msg_set_array_size(int64_t size); + bool is_msg_set(int64_t child_dfo_id) + { + ObLockGuard lock_guard(lock_); + return msg_set_[child_dfo_id]; + } private: static const int64_t MSG_SET_DEFAULT_SIZE = 16; private: diff --git a/src/sql/ob_result_set.cpp b/src/sql/ob_result_set.cpp index bd692e59e3..1e77936774 100644 --- a/src/sql/ob_result_set.cpp +++ b/src/sql/ob_result_set.cpp @@ -254,44 +254,56 @@ int ObResultSet::on_cmd_execute() ret = OB_ERR_UNEXPECTED; LOG_ERROR("invalid inner state", K(cmd_)); } else if (cmd_->cause_implicit_commit()) { - if (my_session_.is_in_transaction() && my_session_.associated_xa()) { - int tmp_ret = OB_SUCCESS; - transaction::ObTxDesc *tx_desc = my_session_.get_tx_desc(); - const transaction::ObXATransID xid = my_session_.get_xid(); - const transaction::ObGlobalTxType global_tx_type = tx_desc->get_global_tx_type(xid); - if (transaction::ObGlobalTxType::XA_TRANS == global_tx_type) { - // commit is not allowed in xa trans - ret = OB_TRANS_XA_ERR_COMMIT; - LOG_WARN("COMMIT is not allowed in a xa trans", K(ret), K(xid), K(global_tx_type), - KPC(tx_desc)); - } else if (transaction::ObGlobalTxType::DBLINK_TRANS == global_tx_type) { - transaction::ObTransID tx_id; - if (OB_FAIL(ObTMService::tm_commit(get_exec_context(), tx_id))) { - LOG_WARN("fail to do commit for dblink trans", K(ret), K(tx_id), K(xid), - K(global_tx_type)); - } - my_session_.restore_auto_commit(); - const bool force_disconnect = false; - if (OB_UNLIKELY(OB_SUCCESS != (tmp_ret = my_session_.get_dblink_context().clean_dblink_conn(force_disconnect)))) { - LOG_WARN("dblink transaction failed to release dblink connections", K(tmp_ret), K(tx_id), K(xid)); - } - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected global trans type", K(ret), K(xid), K(global_tx_type), KPC(tx_desc)); + if (OB_FAIL(implicit_commit_before_cmd_execute(my_session_, + get_exec_context(), + cmd_->get_cmd_type()))) { + LOG_WARN("failed to implicit commit before cmd execute", K(ret)); + } + } + return ret; +} + +int ObResultSet::implicit_commit_before_cmd_execute(ObSQLSessionInfo &session_info, + ObExecContext &exec_ctx, + const int cmd_type) +{ + int ret = OB_SUCCESS; + if (session_info.is_in_transaction() && session_info.associated_xa()) { + int tmp_ret = OB_SUCCESS; + transaction::ObTxDesc *tx_desc = session_info.get_tx_desc(); + const transaction::ObXATransID xid = session_info.get_xid(); + const transaction::ObGlobalTxType global_tx_type = tx_desc->get_global_tx_type(xid); + if (transaction::ObGlobalTxType::XA_TRANS == global_tx_type) { + // commit is not allowed in xa trans + ret = OB_TRANS_XA_ERR_COMMIT; + LOG_WARN("COMMIT is not allowed in a xa trans", K(ret), K(xid), K(global_tx_type), + KPC(tx_desc)); + } else if (transaction::ObGlobalTxType::DBLINK_TRANS == global_tx_type) { + transaction::ObTransID tx_id; + if (OB_FAIL(ObTMService::tm_commit(exec_ctx, tx_id))) { + LOG_WARN("fail to do commit for dblink trans", K(ret), K(tx_id), K(xid), + K(global_tx_type)); + } + session_info.restore_auto_commit(); + const bool force_disconnect = false; + if (OB_UNLIKELY(OB_SUCCESS != (tmp_ret = session_info.get_dblink_context().clean_dblink_conn(force_disconnect)))) { + LOG_WARN("dblink transaction failed to release dblink connections", K(tmp_ret), K(tx_id), K(xid)); } - get_exec_context().set_need_disconnect(false); } else { - // implicit end transaction and start transaction will not clear next scope transaction settings by: - // a. set by `set transaction read only` - // b. set by `set transaction isolation level XXX` - const int cmd_type = cmd_->get_cmd_type(); - bool keep_trans_variable = (cmd_type == stmt::T_START_TRANS); - if (OB_FAIL(ObSqlTransControl::implicit_end_trans(get_exec_context(), false, NULL, !keep_trans_variable))) { - LOG_WARN("fail end implicit trans on cmd execute", K(ret)); - } else if (my_session_.need_recheck_txn_readonly() && my_session_.get_tx_read_only()) { - ret = OB_ERR_CANT_EXECUTE_IN_READ_ONLY_TRANSACTION; - LOG_WARN("cmd can not execute because txn is read only", K(ret), K(cmd_type)); - } + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected global trans type", K(ret), K(xid), K(global_tx_type), KPC(tx_desc)); + } + exec_ctx.set_need_disconnect(false); + } else { + // implicit end transaction and start transaction will not clear next scope transaction settings by: + // a. set by `set transaction read only` + // b. set by `set transaction isolation level XXX` + bool keep_trans_variable = (cmd_type == stmt::T_START_TRANS); + if (OB_FAIL(ObSqlTransControl::implicit_end_trans(exec_ctx, false, NULL, !keep_trans_variable))) { + LOG_WARN("fail end implicit trans on cmd execute", K(ret)); + } else if (session_info.need_recheck_txn_readonly() && session_info.get_tx_read_only()) { + ret = OB_ERR_CANT_EXECUTE_IN_READ_ONLY_TRANSACTION; + LOG_WARN("cmd can not execute because txn is read only", K(ret), K(cmd_type)); } } return ret; diff --git a/src/sql/ob_result_set.h b/src/sql/ob_result_set.h index 3046feb597..af4ad7a941 100644 --- a/src/sql/ob_result_set.h +++ b/src/sql/ob_result_set.h @@ -333,6 +333,9 @@ public: obmysql::ObMySQLField &mfield); void set_close_fail_callback(ObFunction func) { close_fail_cb_ = func; } void set_will_retry() { will_retry_ = true; } + static int implicit_commit_before_cmd_execute(ObSQLSessionInfo &session_info, + ObExecContext &exec_ctx, + const int cmd_type); private: // types and constants static const int64_t TRANSACTION_SET_VIOLATION_MAX_RETRY = 3; diff --git a/src/sql/ob_sql_utils.cpp b/src/sql/ob_sql_utils.cpp index 2327bcb46b..5ab03ae6f6 100644 --- a/src/sql/ob_sql_utils.cpp +++ b/src/sql/ob_sql_utils.cpp @@ -5314,18 +5314,36 @@ bool ObSQLUtils::is_external_files_on_local_disk(const ObString &url) int ObSQLUtils::split_remote_object_storage_url(ObString &url, ObBackupStorageInfo &storage_info) { int ret = OB_SUCCESS; - ObString access_id; - ObString access_key; + ObString https_header = "https://"; + ObString http_header = "http://"; + ObString access_id = url.split_on(':').trim_space_only(); + ObString access_key = url.split_on('@').trim_space_only(); ObString host_name; - ObString access_info = url.split_on('/').trim_space_only(); - if (access_info.empty()) { + int64_t header_len = 0; + + url = url.trim_space_only(); + if (url.prefix_match_ci(https_header)) { + header_len = https_header.length(); + } else if (url.prefix_match_ci(http_header)) { + header_len = http_header.length(); + } else { + header_len = 0; + } + if (header_len > 0) { + host_name = url; + url += header_len; + ObString temp = url.split_on('/'); + host_name.assign_ptr(host_name.ptr(), header_len + temp.length()); + host_name = host_name.trim_space_only(); + } else { + host_name = url.split_on('/').trim_space_only(); + } + url = url.trim_space_only(); + if (access_id.empty() || access_key.empty() || host_name.empty() || url.empty()) { ret = OB_URI_ERROR; LOG_WARN("incorrect uri", K(ret)); - } else { - access_id = access_info.split_on(':').trim_space_only(); - access_key = access_info.split_on('@').trim_space_only(); - host_name = access_info.trim_space_only(); } + LOG_DEBUG("check access info", K(access_id), K(access_key), K(host_name), K(url)); //fill storage_info if (OB_SUCC(ret)) { diff --git a/src/sql/optimizer/ob_conflict_detector.cpp b/src/sql/optimizer/ob_conflict_detector.cpp index 873c72eae4..6b108f8612 100644 --- a/src/sql/optimizer/ob_conflict_detector.cpp +++ b/src/sql/optimizer/ob_conflict_detector.cpp @@ -788,11 +788,11 @@ int ObConflictDetectorGenerator::generate_outer_join_detectors(const ObDMLStmt * ObSEArray detectors; if (OB_FAIL(flatten_inner_join(table_item, table_filter, table_items))) { LOG_WARN("failed to flatten inner join", K(ret)); - } else if (OB_FAIL(generate_inner_join_detectors(stmt, + } else if (OB_FAIL(SMART_CALL(generate_inner_join_detectors(stmt, table_items, table_filter, baserels, - detectors))) { + detectors)))) { LOG_WARN("failed to generate inner join detectors", K(ret)); } else if (OB_FAIL(append(outer_join_detectors, detectors))) { LOG_WARN("failed to append detectors", K(ret)); @@ -907,20 +907,20 @@ int ObConflictDetectorGenerator::inner_generate_outer_join_detectors(const ObDML join_quals))) { LOG_WARN("failed to pushdown on conditions", K(ret)); //3. generate left child detectors - } else if (OB_FAIL(generate_outer_join_detectors(stmt, + } else if (OB_FAIL(SMART_CALL(generate_outer_join_detectors(stmt, joined_table->left_table_, left_quals, baserels, - left_detectors))) { + left_detectors)))) { LOG_WARN("failed to generate outer join detectors", K(ret)); } else if (OB_FAIL(append(outer_join_detectors, left_detectors))) { LOG_WARN("failed to append detectors", K(ret)); //4. generate right child detectors - } else if (OB_FAIL(generate_outer_join_detectors(stmt, + } else if (OB_FAIL(SMART_CALL(generate_outer_join_detectors(stmt, joined_table->right_table_, right_quals, baserels, - right_detectors))) { + right_detectors)))) { LOG_WARN("failed to generate outer join detectors", K(ret)); } else if (OB_FAIL(append(outer_join_detectors, right_detectors))) { LOG_WARN("failed to append detectors", K(ret)); diff --git a/src/sql/optimizer/ob_del_upd_log_plan.cpp b/src/sql/optimizer/ob_del_upd_log_plan.cpp index d0051c265e..c0294b0c8f 100644 --- a/src/sql/optimizer/ob_del_upd_log_plan.cpp +++ b/src/sql/optimizer/ob_del_upd_log_plan.cpp @@ -1179,7 +1179,9 @@ int ObDelUpdLogPlan::create_pdml_insert_plan(ObLogicalOperator *&top, } else if (OB_FAIL(allocate_exchange_as_top(top, exch_info))) { LOG_WARN("failed to allocate exchange as top", K(ret)); } else if (osg_info != NULL && - OB_FAIL(allocate_optimizer_stats_gathering_as_top(top, *osg_info))) { + OB_FAIL(allocate_optimizer_stats_gathering_as_top(top, + *osg_info, + OSG_TYPE::GATHER_OSG))) { LOG_WARN("failed to allocate optimizer stats gathering"); } else if (OB_FAIL(allocate_pdml_insert_as_top(top, is_index_maintenance, @@ -1194,7 +1196,8 @@ int ObDelUpdLogPlan::create_pdml_insert_plan(ObLogicalOperator *&top, } int ObDelUpdLogPlan::allocate_optimizer_stats_gathering_as_top(ObLogicalOperator *&old_top, - OSGShareInfo &info) + OSGShareInfo &info, + OSG_TYPE type) { int ret = OB_SUCCESS; ObLogOptimizerStatsGathering *osg = NULL; @@ -1206,8 +1209,6 @@ int ObDelUpdLogPlan::allocate_optimizer_stats_gathering_as_top(ObLogicalOperator ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to allocate sequence operator", K(ret)); } else { - OSG_TYPE type = old_top->need_osg_merge() ? OSG_TYPE::MERGE_OSG : - old_top->is_sharding() ? OSG_TYPE::GATHER_OSG : OSG_TYPE::NORMAL_OSG; osg->set_child(ObLogicalOperator::first_child, old_top); osg->set_osg_type(type); osg->set_table_id(info.table_id_); diff --git a/src/sql/optimizer/ob_del_upd_log_plan.h b/src/sql/optimizer/ob_del_upd_log_plan.h index 838dc5da9d..697c3efee6 100644 --- a/src/sql/optimizer/ob_del_upd_log_plan.h +++ b/src/sql/optimizer/ob_del_upd_log_plan.h @@ -245,7 +245,8 @@ protected: virtual int generate_normal_raw_plan() override; virtual int generate_dblink_raw_plan() override; int allocate_optimizer_stats_gathering_as_top(ObLogicalOperator *&old_top, - OSGShareInfo &info); + OSGShareInfo &info, + OSG_TYPE type); private: DISALLOW_COPY_AND_ASSIGN(ObDelUpdLogPlan); diff --git a/src/sql/optimizer/ob_dynamic_sampling.cpp b/src/sql/optimizer/ob_dynamic_sampling.cpp index f85728ac26..6a4b3600b0 100644 --- a/src/sql/optimizer/ob_dynamic_sampling.cpp +++ b/src/sql/optimizer/ob_dynamic_sampling.cpp @@ -492,6 +492,7 @@ int ObDynamicSampling::estimte_rowcount(int64_t max_ds_timeout, ObSqlString raw_sql_str; ObSqlString sample_str; ObSqlString basic_hint_str; + ObSqlString table_str; bool is_no_backslash_escapes = false; int64_t nested_count = -1; sql::ObSQLSessionInfo::StmtSavedValue *session_value = NULL; @@ -499,10 +500,12 @@ int ObDynamicSampling::estimte_rowcount(int64_t max_ds_timeout, ObSQLSessionInfo *session_info = ctx_->get_session_info(); bool need_restore_session = false; transaction::ObTxDesc *tx_desc = NULL; - if (OB_FAIL(add_block_sample_info(sample_block_ratio_, seed_, sample_str))) { + if (!is_big_table_ && OB_FAIL(add_block_sample_info(sample_block_ratio_, seed_, sample_str))) { LOG_WARN("failed to add block sample info", K(ret)); - } else if (OB_FAIL(add_basic_hint_info(basic_hint_str, max_ds_timeout, degree))) { + } else if (OB_FAIL(add_basic_hint_info(basic_hint_str, max_ds_timeout, is_big_table_ ? 1 : degree))) { LOG_WARN("failed to add basic hint info", K(ret)); + } else if (OB_FAIL(add_table_clause(table_str))) { + LOG_WARN("failed to add table clause", K(ret)); } else if (OB_FAIL(pack(raw_sql_str))) { LOG_WARN("failed to pack dynamic sampling", K(ret)); } else if (OB_FAIL(prepare_and_store_session(session_info, session_value, @@ -602,24 +605,14 @@ int ObDynamicSampling::pack(ObSqlString &raw_sql_str) if (OB_FAIL(gen_select_filed(select_fields))) { LOG_WARN("failed to generate select filed", K(ret)); } else if (OB_FAIL(raw_sql_str.append_fmt(lib::is_oracle_mode() ? - "SELECT %.*s %.*s FROM \"%.*s\".\"%.*s\" %.*s %.*s %s%.*s%s %s %.*s" : - "SELECT %.*s %.*s FROM `%.*s`.`%.*s` %.*s %.*s %s%.*s%s %s %.*s" , + "SELECT %.*s %.*s FROM %.*s %s %.*s" : + "SELECT %.*s %.*s FROM %.*s %s %.*s" , basic_hints_.length(), basic_hints_.ptr(), static_cast(select_fields.length()), select_fields.ptr(), - db_name_.length(), - db_name_.ptr(), - table_name_.length(), - table_name_.ptr(), - partition_list_.length(), - partition_list_.ptr(), - sample_block_.length(), - sample_block_.ptr(), - alias_name_.empty() ? " " : (lib::is_oracle_mode() ? "\"" : "`"), - alias_name_.length(), - alias_name_.ptr(), - alias_name_.empty() ? " " : (lib::is_oracle_mode() ? "\"" : "`"), + table_clause_.length(), + table_clause_.ptr(), where_conditions_.empty() ? " " : "WHERE", where_conditions_.length(), where_conditions_.ptr()))) { @@ -779,37 +772,48 @@ int ObDynamicSampling::calc_table_sample_block_ratio(const ObDSTableParam ¶m if (param.is_virtual_table_) { sample_block_ratio_ = 100.0; seed_ = param.degree_ > 1 ? 0 : 1; - } else if (OB_UNLIKELY((sample_micro_cnt = get_dynamic_sampling_micro_block_num(param)) < 1)) { + } else if (OB_UNLIKELY((sample_micro_cnt = get_dynamic_sampling_micro_block_num(param)) < 1 || + (micro_block_num_ > 0 && memtable_row_count_ + sstable_row_count_ <= 0))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(param)); } else if (OB_FAIL(estimate_table_block_count_and_row_count(param))) { LOG_WARN("failed to estimate table block count and row count", K(ret)); } else { - //1.retire to memtable sample - if (sstable_row_count_ < memtable_row_count_) { - double sample_row_cnt = MAGIC_MAX_AUTO_SAMPLE_SIZE * (1.0 * sample_micro_cnt / OB_DS_BASIC_SAMPLE_MICRO_CNT); - if (memtable_row_count_ < sample_row_cnt) { - sample_block_ratio_ = 100.0; - } else { - sample_block_ratio_ = 100.0 * sample_row_cnt / memtable_row_count_; - } - } else { - //2.use the block sample - if (sample_micro_cnt >= micro_block_num_) { - sample_block_ratio_ = 100.0; - } else { - sample_block_ratio_ = 100.0 * sample_micro_cnt / micro_block_num_; - } - } - //3.try adjust sample block ratio according to the degree - if (param.degree_ > 1 && sample_block_ratio_ < 100.0) {//adjust sample ratio according to the degree. - sample_block_ratio_ = sample_block_ratio_ * param.degree_; + int64_t max_allowed_multiple = sample_micro_cnt > OB_DS_BASIC_SAMPLE_MICRO_CNT ? sample_micro_cnt / OB_DS_BASIC_SAMPLE_MICRO_CNT : 1; + if (micro_block_num_ > OB_DS_MAX_BASIC_SAMPLE_MICRO_CNT * max_allowed_multiple && + MAGIC_MAX_AUTO_SAMPLE_SIZE * max_allowed_multiple < memtable_row_count_ + sstable_row_count_) { + is_big_table_ = true; + sample_big_table_rown_cnt_ = MAGIC_MAX_AUTO_SAMPLE_SIZE * max_allowed_multiple; + sample_block_ratio_ = 100.0 * sample_big_table_rown_cnt_ / (memtable_row_count_ + sstable_row_count_); sample_block_ratio_ = sample_block_ratio_ < 100.0 ? sample_block_ratio_ : 100.0; + } else { + //1.retire to memtable sample + if (sstable_row_count_ < memtable_row_count_) { + double sample_row_cnt = MAGIC_MAX_AUTO_SAMPLE_SIZE * (1.0 * sample_micro_cnt / OB_DS_BASIC_SAMPLE_MICRO_CNT); + if (memtable_row_count_ < sample_row_cnt) { + sample_block_ratio_ = 100.0; + } else { + sample_block_ratio_ = 100.0 * sample_row_cnt / (memtable_row_count_ + sstable_row_count_); + } + } else { + //2.use the block sample + if (sample_micro_cnt >= micro_block_num_) { + sample_block_ratio_ = 100.0; + } else { + sample_block_ratio_ = 100.0 * sample_micro_cnt / micro_block_num_; + } + } + //3.try adjust sample block ratio according to the degree + if (param.degree_ > 1 && sample_block_ratio_ < 100.0) {//adjust sample ratio according to the degree. + sample_block_ratio_ = sample_block_ratio_ * param.degree_; + sample_block_ratio_ = sample_block_ratio_ < 100.0 ? sample_block_ratio_ : 100.0; + } + sample_block_ratio_ = sample_block_ratio_ < 0.000001 ? 0.000001 : sample_block_ratio_; + //4.adjust the seed. + seed_ = (param.degree_ > 1 || param.partition_infos_.count() > 1) ? 0 : 1; } - //4.adjust the seed. - seed_ = (param.degree_ > 1 || param.partition_infos_.count() > 1) ? 0 : 1; } - LOG_TRACE("succeed to calc table sample block ratio", K(param), K(seed_), K(sample_micro_cnt), + LOG_TRACE("succeed to calc table sample block ratio", K(param), K(seed_), K(sample_micro_cnt), K(is_big_table_), K(sample_block_ratio_), K(micro_block_num_), K(sstable_row_count_), K(memtable_row_count_)); return ret; @@ -1229,6 +1233,50 @@ bool ObDynamicSampling::all_ds_col_stats_are_gathered(const ObDSTableParam ¶ return res; } +int ObDynamicSampling::add_table_clause(ObSqlString &table_str) +{ + int ret = OB_SUCCESS; + int64_t big_table_row = 100000; + if (OB_UNLIKELY(is_big_table_ && sample_big_table_rown_cnt_ < MAGIC_MAX_AUTO_SAMPLE_SIZE)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(sample_big_table_rown_cnt_), K(is_big_table_)); + } else if (is_big_table_ && OB_FAIL(table_str.append_fmt(lib::is_oracle_mode() ? "(SELECT * FROM \"%.*s\".\"%.*s\" %.*s %.*s FETCH NEXT %ld ROWS ONLY) %s%.*s%s" : + "(SELECT * FROM `%.*s`.`%.*s` %.*s %.*s LIMIT %ld) %s%.*s%s" , + db_name_.length(), + db_name_.ptr(), + table_name_.length(), + table_name_.ptr(), + partition_list_.length(), + partition_list_.ptr(), + sample_block_.length(), + sample_block_.ptr(), + sample_big_table_rown_cnt_, + alias_name_.empty() ? " " : (lib::is_oracle_mode() ? "\"" : "`"), + alias_name_.length(), + alias_name_.ptr(), + alias_name_.empty() ? " " : (lib::is_oracle_mode() ? "\"" : "`")))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (!is_big_table_ && OB_FAIL(table_str.append_fmt(lib::is_oracle_mode() ? "\"%.*s\".\"%.*s\" %.*s %.*s %s%.*s%s" : + "`%.*s`.`%.*s` %.*s %.*s %s%.*s%s" , + db_name_.length(), + db_name_.ptr(), + table_name_.length(), + table_name_.ptr(), + partition_list_.length(), + partition_list_.ptr(), + sample_block_.length(), + sample_block_.ptr(), + alias_name_.empty() ? " " : (lib::is_oracle_mode() ? "\"" : "`"), + alias_name_.length(), + alias_name_.ptr(), + alias_name_.empty() ? " " : (lib::is_oracle_mode() ? "\"" : "`")))) { + LOG_WARN("failed to append fmt", K(ret)); + } else { + table_clause_ = table_str.string(); + } + return ret; +} + int ObDynamicSamplingUtils::get_valid_dynamic_sampling_level(const ObSQLSessionInfo *session_info, const ObTableDynamicSamplingHint *table_ds_hint, const int64_t global_ds_level, diff --git a/src/sql/optimizer/ob_dynamic_sampling.h b/src/sql/optimizer/ob_dynamic_sampling.h index dc9449ce0a..dc549573a2 100644 --- a/src/sql/optimizer/ob_dynamic_sampling.h +++ b/src/sql/optimizer/ob_dynamic_sampling.h @@ -200,6 +200,7 @@ static T *copy_ds_stat_item(ObIAllocator &allocator, const T &src) const int64_t OB_DS_BASIC_SAMPLE_MICRO_CNT = 32; const int64_t OB_DS_MAX_FILTER_EXPR_COUNT = 10000; const int64_t OB_DS_MIN_QUERY_TIMEOUT = 1000;//Dynamic sampling requires a minimum timeout of 1ms. +const int64_t OB_DS_MAX_BASIC_SAMPLE_MICRO_CNT = 1000000; //const int64_t OB_OPT_DS_ADAPTIVE_SAMPLE_MICRO_CNT = 200; //const int64_t OB_OPT_DS_MAX_TIMES = 7; @@ -223,7 +224,10 @@ public: basic_hints_(), where_conditions_(), ds_stat_items_(), - results_() + results_(), + is_big_table_(false), + sample_big_table_rown_cnt_(0), + table_clause_() {} int estimate_table_rowcount(const ObDSTableParam ¶m, @@ -318,6 +322,7 @@ private: int64_t nested_count, bool is_no_backslash_escapes, transaction::ObTxDesc *tx_desc); + int add_table_clause(ObSqlString &table_str); private: ObOptimizerContext *ctx_; @@ -337,6 +342,9 @@ private: ObString where_conditions_; ObSEArray ds_stat_items_; ObSEArray results_; + bool is_big_table_; + int64_t sample_big_table_rown_cnt_; + ObString table_clause_; //following members will be used for dynamic sampling join in the future //ObString join_type_; //ObString join_conditions_; diff --git a/src/sql/optimizer/ob_insert_log_plan.cpp b/src/sql/optimizer/ob_insert_log_plan.cpp index 4bdc5c1a18..d92a24cc9b 100644 --- a/src/sql/optimizer/ob_insert_log_plan.cpp +++ b/src/sql/optimizer/ob_insert_log_plan.cpp @@ -498,6 +498,31 @@ int ObInsertLogPlan::build_lock_row_flag_expr(ObConstRawExpr *&lock_row_flag_exp return ret; } +int ObInsertLogPlan::get_osg_type(bool is_multi_part_dml, + ObShardingInfo *insert_table_sharding, + int64_t distributed_method, + OSG_TYPE &type) +{ + int ret = OB_SUCCESS; + type = OSG_TYPE::NORMAL_OSG; + if (DIST_PARTITION_WISE == distributed_method || + DIST_PULL_TO_LOCAL == distributed_method) { + //need merge stats + type = OSG_TYPE::GATHER_OSG; + } else if (DIST_BASIC_METHOD == distributed_method) { + if (is_multi_part_dml) { + type = OSG_TYPE::NORMAL_OSG; + } else if (OB_ISNULL(insert_table_sharding)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null sharding", K(ret)); + } else if (insert_table_sharding->is_remote()) { + //need merge stats + type = OSG_TYPE::GATHER_OSG; + } + } + return ret; +} + int ObInsertLogPlan::create_insert_plans(ObIArray &candi_plans, ObTablePartitionInfo *insert_table_part, ObShardingInfo *insert_table_sharding, @@ -511,6 +536,7 @@ int ObInsertLogPlan::create_insert_plans(ObIArray &candi_plans, ObExchangeInfo exch_info; bool is_multi_part_dml = false; CandidatePlan candi_plan; + OSG_TYPE osg_type = OSG_TYPE::NORMAL_OSG; int64_t inherit_sharding_index = OB_INVALID_INDEX; ObShardingInfo *insert_op_sharding = NULL; ObSEArray input_shardings; @@ -533,9 +559,16 @@ int ObInsertLogPlan::create_insert_plans(ObIArray &candi_plans, LOG_WARN("failed to get best insert plan method", K(ret)); } else if (0 == distributed_methods) { /*do nothing*/ + } else if (osg_info != NULL && + OB_FAIL(get_osg_type(is_multi_part_dml, + insert_table_sharding, + distributed_methods, + osg_type))) { + LOG_WARN("failed to get osg type", K(ret)); } else if (osg_info != NULL && OB_FAIL(allocate_optimizer_stats_gathering_as_top(candi_plan.plan_tree_, - *osg_info))) { + *osg_info, + osg_type))) { LOG_WARN("failed to allocate sequence as top", K(ret)); } else if (DIST_PULL_TO_LOCAL == distributed_methods) { if (OB_FAIL(allocate_exchange_as_top(candi_plan.plan_tree_, exch_info))) { @@ -1598,7 +1631,8 @@ int ObInsertLogPlan::candi_allocate_optimizer_stats_merge(OSGShareInfo *osg_info LOG_WARN("failed to allocate exchange as top", K(ret)); } else if (OB_FAIL(allocate_optimizer_stats_gathering_as_top( best_candidates.at(i).plan_tree_, - *osg_info))) { + *osg_info, + OSG_TYPE::MERGE_OSG))) { LOG_WARN("failed to allocate sequence as top", K(ret)); } } diff --git a/src/sql/optimizer/ob_insert_log_plan.h b/src/sql/optimizer/ob_insert_log_plan.h index 4b6ffc72dc..4eadbd3a84 100644 --- a/src/sql/optimizer/ob_insert_log_plan.h +++ b/src/sql/optimizer/ob_insert_log_plan.h @@ -71,6 +71,11 @@ protected: int candi_allocate_pdml_insert(OSGShareInfo *osg_info); int candi_allocate_optimizer_stats_merge(OSGShareInfo *osg_info); + int get_osg_type(bool is_multi_part_dml, + ObShardingInfo *insert_table_sharding, + int64_t distributed_method, + OSG_TYPE &type); + virtual int get_best_insert_dist_method(ObLogicalOperator &top, ObTablePartitionInfo *insert_table_partition, ObShardingInfo *insert_table_sharding, diff --git a/src/sql/optimizer/ob_join_order.cpp b/src/sql/optimizer/ob_join_order.cpp index 92b07a76df..918ea8144b 100644 --- a/src/sql/optimizer/ob_join_order.cpp +++ b/src/sql/optimizer/ob_join_order.cpp @@ -4029,7 +4029,7 @@ int ObJoinOrder::get_candi_range_expr(const ObIArray &range_columns, min_cost_range_count, cost))) { LOG_WARN("failed to calculate range expr cost", K(ret)); - } else if (cost >= min_cost) { + } else if (cost >= min_cost && min_cost_range_count > 500) { //increase cost, ignore in expr range_exprs.pop_back(); if (OB_FAIL(ignore_predicates.push_back(min_cost_in_expr))) { diff --git a/src/sql/optimizer/ob_log_temp_table_insert.cpp b/src/sql/optimizer/ob_log_temp_table_insert.cpp index 24924397c4..10c66bc388 100644 --- a/src/sql/optimizer/ob_log_temp_table_insert.cpp +++ b/src/sql/optimizer/ob_log_temp_table_insert.cpp @@ -111,6 +111,7 @@ int ObLogTempTableInsert::do_re_est_cost(EstimateCostInfo ¶m, double &card, child->get_width(), opt_ctx); cost += op_cost; + card = 0; } return ret; } @@ -129,4 +130,4 @@ int ObLogTempTableInsert::get_plan_item_info(PlanText &plan_text, plan_item.object_alias_len_); } return ret; -} \ No newline at end of file +} diff --git a/src/sql/optimizer/ob_select_log_plan.cpp b/src/sql/optimizer/ob_select_log_plan.cpp index ed6fa20b84..aeb2a8276e 100644 --- a/src/sql/optimizer/ob_select_log_plan.cpp +++ b/src/sql/optimizer/ob_select_log_plan.cpp @@ -389,6 +389,7 @@ int ObSelectLogPlan::get_valid_aggr_algo(const ObIArray &group_by_ex bool &normal_sort_valid) { int ret = OB_SUCCESS; + bool has_keep_aggr = false; if (ignore_hint) { use_hash_valid = true; use_merge_valid = true; @@ -403,10 +404,13 @@ int ObSelectLogPlan::get_valid_aggr_algo(const ObIArray &group_by_ex if (OB_ISNULL(get_stmt()) || OB_ISNULL(optimizer_context_.get_query_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(get_stmt()), K(optimizer_context_.get_query_ctx()), K(ret)); + } else if (OB_FAIL(check_aggr_with_keep(get_stmt()->get_aggr_items(), has_keep_aggr))) { + LOG_WARN("failed to check aggr with keep", K(ret)); } else if (get_stmt()->has_rollup() || group_by_exprs.empty() - || get_stmt()->has_distinct_or_concat_agg()) { - //group_concat and distinct aggregation hold all input rows temporary, + || get_stmt()->has_distinct_or_concat_agg() + || has_keep_aggr) { + //keep_aggr、group_concat and distinct aggregation hold all input rows temporary, //too much memory consumption for hash aggregate. use_hash_valid = false; } @@ -7924,5 +7928,21 @@ int ObSelectLogPlan::candi_allocate_order_by_if_losted(ObIArray &orde return ret; } +int ObSelectLogPlan::check_aggr_with_keep(const ObIArray &aggr_items, + bool &has_keep_aggr) +{ + int ret = OB_SUCCESS; + has_keep_aggr = false; + for (int64_t i = 0; OB_SUCC(ret) && !has_keep_aggr && i < aggr_items.count(); ++i) { + if (OB_ISNULL(aggr_items.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("aggr item is null", K(ret)); + } else if (IS_KEEP_AGGR_FUN(aggr_items.at(i)->get_expr_type())) { + has_keep_aggr = true; + } + } + return ret; +} + }//sql }//oceanbase diff --git a/src/sql/optimizer/ob_select_log_plan.h b/src/sql/optimizer/ob_select_log_plan.h index 34627359bc..eb000756d0 100644 --- a/src/sql/optimizer/ob_select_log_plan.h +++ b/src/sql/optimizer/ob_select_log_plan.h @@ -930,6 +930,7 @@ int generate_window_functions_plan(WinFuncOpHelper &win_func_helper, int contain_enum_set_rowkeys(const ObLogTableScan &table_scan, bool &contain); int candi_allocate_order_by_if_losted(ObIArray &order_items); + int check_aggr_with_keep(const ObIArray& aggr_items, bool &has_keep_aggr); DISALLOW_COPY_AND_ASSIGN(ObSelectLogPlan); }; diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 3b8305c81d..2688a41584 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -4880,12 +4880,12 @@ opt_table_option_list opt_partition_option with_column_group | create_with_opt_hint special_table_type TABLE opt_if_not_exists relation_factor '(' table_element_list ')' opt_table_option_list opt_partition_option select_stmt { - (void)($1); + (void)($10); ParseNode *table_elements = NULL; ParseNode *table_options = NULL; merge_nodes(table_elements, result, T_TABLE_ELEMENT_LIST, $7); merge_nodes(table_options, result, T_TABLE_OPTION_LIST, $9); - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 9, + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 10, $2, /* temporary option */ $4, /* if not exists */ $5, /* table name */ @@ -4894,19 +4894,19 @@ opt_table_option_list opt_partition_option with_column_group $10, /* partition optition */ NULL, /* column group */ NULL, /* oracle兼容模式下存放临时表的 on commit 选项 */ - $11); /* select_stmt */ + $11, /* select_stmt */ + $1); /* hints */ $$->reserved_ = 0; } | create_with_opt_hint special_table_type TABLE opt_if_not_exists relation_factor '(' table_element_list ')' opt_table_option_list opt_partition_option AS select_stmt { - (void)($1); (void)$11; ParseNode *table_elements = NULL; ParseNode *table_options = NULL; merge_nodes(table_elements, result, T_TABLE_ELEMENT_LIST, $7); merge_nodes(table_options, result, T_TABLE_OPTION_LIST, $9); - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 9, + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 10, $2, /* temporary option */ $4, /* if not exists */ $5, /* table name */ @@ -4915,19 +4915,19 @@ opt_table_option_list opt_partition_option with_column_group $10, /* partition optition */ NULL, /* column group */ NULL, /* oracle兼容模式下存放临时表的 on commit 选项 */ - $12); /* select_stmt */ + $12, /* select_stmt */ + $1); /* hints */ $$->reserved_ = 0; } | create_with_opt_hint special_table_type TABLE opt_if_not_exists relation_factor '(' table_element_list ')' opt_table_option_list opt_partition_option with_column_group opt_as select_stmt { - (void)($1); (void)$12; ParseNode *table_elements = NULL; ParseNode *table_options = NULL; merge_nodes(table_elements, result, T_TABLE_ELEMENT_LIST, $7); merge_nodes(table_options, result, T_TABLE_OPTION_LIST, $9); - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 9, + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 10, $2, /* temporary option */ $4, /* if not exists */ $5, /* table name */ @@ -4936,15 +4936,15 @@ opt_table_option_list opt_partition_option with_column_group $10, /* partition optition */ $11, /* column group */ NULL, /* oracle兼容模式下存放临时表的 on commit 选项 */ - $13); /* select_stmt */ + $13, /* select_stmt */ + $1); /* hints */ $$->reserved_ = 0; } | create_with_opt_hint special_table_type TABLE opt_if_not_exists relation_factor table_option_list opt_partition_option select_stmt { - (void)($1); ParseNode *table_options = NULL; merge_nodes(table_options, result, T_TABLE_OPTION_LIST, $6); - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 9, + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 10, $2, /* temporary option */ $4, /* if not exists */ $5, /* table name */ @@ -4953,16 +4953,16 @@ opt_table_option_list opt_partition_option with_column_group $7, /* partition optition */ NULL, /* column group */ NULL, /* oracle兼容模式下存放临时表的 on commit 选项 */ - $8); /* select_stmt */ + $8, /* select_stmt */ + $1); $$->reserved_ = 0; } | create_with_opt_hint special_table_type TABLE opt_if_not_exists relation_factor table_option_list opt_partition_option AS select_stmt { - (void)($1); (void)$8; ParseNode *table_options = NULL; merge_nodes(table_options, result, T_TABLE_OPTION_LIST, $6); - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 9, + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 10, $2, /* temporary option */ $4, /* if not exists */ $5, /* table name */ @@ -4971,16 +4971,16 @@ opt_table_option_list opt_partition_option with_column_group $7, /* partition optition */ NULL, /* column group */ NULL, /* oracle兼容模式下存放临时表的 on commit 选项 */ - $9); /* select_stmt */ + $9, /* select_stmt */ + $1); $$->reserved_ = 0; } | create_with_opt_hint special_table_type TABLE opt_if_not_exists relation_factor table_option_list opt_partition_option with_column_group opt_as select_stmt { - (void)($1); (void)$9; ParseNode *table_options = NULL; merge_nodes(table_options, result, T_TABLE_OPTION_LIST, $6); - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 9, + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 10, $2, /* temporary option */ $4, /* if not exists */ $5, /* table name */ @@ -4989,13 +4989,14 @@ opt_table_option_list opt_partition_option with_column_group $7, /* partition optition */ $8, /* column group */ NULL, /* oracle兼容模式下存放临时表的 on commit 选项 */ - $10); /* select_stmt */ + $10, /* select_stmt */ + $1); $$->reserved_ = 0; } | create_with_opt_hint special_table_type TABLE opt_if_not_exists relation_factor partition_option select_stmt { (void)($1); - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 9, + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 10, $2, /* temporary option */ $4, /* if not exists */ $5, /* table name */ @@ -5004,14 +5005,14 @@ opt_table_option_list opt_partition_option with_column_group $6, /* partition optition */ NULL, /* column group */ NULL, /* oracle兼容模式下存放临时表的 on commit 选项 */ - $7); /* select_stmt */ + $7, /* select_stmt */ + $1); $$->reserved_ = 1; /* mean partition optition is partition_option, not opt_partition_option*/ } | create_with_opt_hint special_table_type TABLE opt_if_not_exists relation_factor partition_option AS select_stmt { - (void)($1); (void)$7; - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 9, + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 10, $2, /* temporary option */ $4, /* if not exists */ $5, /* table name */ @@ -5020,14 +5021,14 @@ opt_table_option_list opt_partition_option with_column_group $6, /* partition optition */ NULL, /* column group */ NULL, /* oracle兼容模式下存放临时表的 on commit 选项 */ - $8); /* select_stmt */ + $8, /* select_stmt */ + $1); $$->reserved_ = 1; /* mean partition optition is partition_option, not opt_partition_option*/ } | create_with_opt_hint special_table_type TABLE opt_if_not_exists relation_factor partition_option with_column_group opt_as select_stmt { - (void)($1); (void)$8; - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 9, + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 10, $2, /* temporary option */ $4, /* if not exists */ $5, /* table name */ @@ -5036,13 +5037,13 @@ opt_table_option_list opt_partition_option with_column_group $6, /* partition optition */ $7, /* column group */ NULL, /* oracle兼容模式下存放临时表的 on commit 选项 */ - $9); /* select_stmt */ + $9, /* select_stmt */ + $1); $$->reserved_ = 1; /* mean partition optition is partition_option, not opt_partition_option*/ } | create_with_opt_hint special_table_type TABLE opt_if_not_exists relation_factor select_stmt { - (void)($1); - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 9, + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 10, $2, /* temporary option */ $4, /* if not exists */ $5, /* table name */ @@ -5051,13 +5052,13 @@ opt_table_option_list opt_partition_option with_column_group NULL, /* partition optition */ NULL, /* column group */ NULL, /* oracle兼容模式下存放临时表的 on commit 选项 */ - $6); /* select_stmt */ + $6, /* select_stmt */ + $1); /* hints */ $$->reserved_ = 0; } | create_with_opt_hint special_table_type TABLE opt_if_not_exists relation_factor with_column_group select_stmt { - (void)($1); - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 9, + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 10, $2, /* temporary option */ $4, /* if not exists */ $5, /* table name */ @@ -5066,13 +5067,13 @@ opt_table_option_list opt_partition_option with_column_group NULL, /* partition optition */ $6, /* column group */ NULL, /* oracle兼容模式下存放临时表的 on commit 选项 */ - $7); /* select_stmt */ + $7, /* select_stmt */ + $1); $$->reserved_ = 0; } | create_with_opt_hint special_table_type TABLE opt_if_not_exists relation_factor AS select_stmt { - (void)($1); - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 9, + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 10, $2, /* temporary option */ $4, /* if not exists */ $5, /* table name */ @@ -5081,13 +5082,13 @@ opt_table_option_list opt_partition_option with_column_group NULL, /* partition optition */ NULL, /* column group */ NULL, /* oracle兼容模式下存放临时表的 on commit 选项 */ - $7); /* select_stmt */ + $7, /* select_stmt */ + $1); /* hints */ $$->reserved_ = 0; } | create_with_opt_hint special_table_type TABLE opt_if_not_exists relation_factor with_column_group AS select_stmt { - (void)($1); - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 9, + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_TABLE, 10, $2, /* temporary option */ $4, /* if not exists */ $5, /* table name */ @@ -5096,7 +5097,8 @@ opt_table_option_list opt_partition_option with_column_group NULL, /* partition optition */ $6, /* column group */ NULL, /* oracle兼容模式下存放临时表的 on commit 选项 */ - $8); /* select_stmt */ + $8, /* select_stmt */ + $1); $$->reserved_ = 0; } ; @@ -6957,6 +6959,7 @@ TABLE_MODE opt_equal_mark STRING_VALUE (void) ($$); if ($3->value_ < 0) { yyerror(&@1, result, "value for DELAY_KEY_WRITE shouldn't be negative"); + YYERROR; } else { malloc_non_terminal_node($$, result->malloc_pool_, T_DELAY_KEY_WRITE, 1, $3); } @@ -6966,6 +6969,7 @@ TABLE_MODE opt_equal_mark STRING_VALUE (void)($2); if ($3->value_ < 0) { yyerror(&@1, result, "value for AVG_ROW_LENGTH shouldn't be negative"); + YYERROR; } else { malloc_non_terminal_node($$, result->malloc_pool_, T_AVG_ROW_LENGTH, 1, $3); } @@ -6975,6 +6979,7 @@ TABLE_MODE opt_equal_mark STRING_VALUE (void)($2); if ($3->value_ < 0) { yyerror(&@1, result, "value for CHECKSUM shouldn't be negative"); + YYERROR; } else { malloc_non_terminal_node($$, result->malloc_pool_, T_TABLE_CHECKSUM, 1, $3); } diff --git a/src/sql/resolver/ddl/ob_create_table_resolver.cpp b/src/sql/resolver/ddl/ob_create_table_resolver.cpp index 8dbbc14b3a..0d94c530f1 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_table_resolver.cpp @@ -820,6 +820,13 @@ int ObCreateTableResolver::resolve(const ParseNode &parse_tree) LOG_WARN("failed to deep copy string in part expr"); } } + if (OB_SUCC(ret) && is_create_as_sel) { + if (OB_FAIL(resolve_hints(create_table_node->children_[CREATE_TABLE_AS_SEL_NUM_CHILD - 1], + *create_table_stmt, + create_table_stmt->get_create_table_arg().schema_))) { + LOG_WARN("fail to resolve hint", K(ret)); + } + } } return ret; } @@ -1851,7 +1858,7 @@ int ObCreateTableResolver::resolve_table_elements_from_select(const ParseNode &p int ret = OB_SUCCESS; ObCreateTableStmt *create_table_stmt = static_cast(stmt_); const ObTableSchema *base_table_schema = NULL; - ParseNode *sub_sel_node = parse_tree.children_[CREATE_TABLE_AS_SEL_NUM_CHILD - 1]; + ParseNode *sub_sel_node = parse_tree.children_[CREATE_TABLE_AS_SEL_NUM_CHILD - 2]; ObSelectStmt *select_stmt = NULL; ObSelectResolver select_resolver(params_); select_resolver.params_.is_from_create_table_ = true; @@ -3276,8 +3283,7 @@ int ObCreateTableResolver::resolve_column_group(const ParseNode *cg_node) LOG_WARN("fail to reserve", KR(ret), K(column_cnt)); } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { LOG_WARN("fail to get min data version", KR(ret), K(tenant_id)); - } else if (!(compat_version >= DATA_VERSION_4_3_0_0) - && can_add_column_group(table_schema)) { + } else if (!(compat_version >= DATA_VERSION_4_3_0_0)) { if (OB_NOT_NULL(cg_node) && (T_COLUMN_GROUP == cg_node->type_)) { ret = OB_NOT_SUPPORTED; LOG_WARN("can't support column store if version less than 4_1_0_0", KR(ret), K(compat_version)); @@ -3296,7 +3302,8 @@ int ObCreateTableResolver::resolve_column_group(const ParseNode *cg_node) if (OB_SUCC(ret) && OB_LIKELY(tenant_config.is_valid()) && nullptr == cg_node) { /* force to build each cg*/ - if (OB_FAIL(ObTableStoreFormat::find_table_store_type( + if (!ObSchemaUtils::can_add_column_group(table_schema)) { + } else if (OB_FAIL(ObTableStoreFormat::find_table_store_type( tenant_config->default_table_store_format.get_value_string(), table_store_type))) { LOG_WARN("fail to get table store format", K(ret), K(table_store_type)); @@ -3314,6 +3321,7 @@ int ObCreateTableResolver::resolve_column_group(const ParseNode *cg_node) /* force to build all cg*/ ObColumnGroupSchema all_cg; if (OB_FAIL(ret)) { + } else if (!ObSchemaUtils::can_add_column_group(table_schema)) { } else if (ObTableStoreFormat::is_row_with_column_store(table_store_type)) { bool is_all_cg_exist = false; if (OB_FAIL(table_schema.is_column_group_exist(OB_ALL_COLUMN_GROUP_NAME, is_all_cg_exist))) { @@ -3348,15 +3356,6 @@ int ObCreateTableResolver::resolve_column_group(const ParseNode *cg_node) return ret; } -bool ObCreateTableResolver::can_add_column_group(const ObTableSchema &table_schema) -{ - bool can_add_cg = false; - if (table_schema.is_user_table() - || table_schema.is_tmp_table()) { - can_add_cg = true; - } - return can_add_cg; -} int ObCreateTableResolver::add_inner_index_for_heap_gtt() { int ret = OB_SUCCESS; diff --git a/src/sql/resolver/ddl/ob_create_table_resolver.h b/src/sql/resolver/ddl/ob_create_table_resolver.h index f0417fbf45..1ddb51082b 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver.h +++ b/src/sql/resolver/ddl/ob_create_table_resolver.h @@ -134,7 +134,6 @@ private: uint64_t gen_column_group_id(); int resolve_column_group(const ParseNode *cg_node); - bool can_add_column_group(const ObTableSchema &table_schema); int add_inner_index_for_heap_gtt(); int check_max_row_data_length(const ObTableSchema &table_schema); diff --git a/src/sql/resolver/ddl/ob_create_table_stmt.h b/src/sql/resolver/ddl/ob_create_table_stmt.h index 8badc19862..d01b39b8e0 100644 --- a/src/sql/resolver/ddl/ob_create_table_stmt.h +++ b/src/sql/resolver/ddl/ob_create_table_stmt.h @@ -97,9 +97,9 @@ private: // } // // create table xxx as already_exist_table, pay attention to whether data are need - protected: - ObSelectStmt *sub_select_stmt_; //create table ... as select... - ObSelectStmt *view_define_; +protected: + ObSelectStmt *sub_select_stmt_; //create table ... as select... + ObSelectStmt *view_define_; }; inline obrpc::ObCreateTableArg &ObCreateTableStmt::get_create_table_arg() diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index 2cfda696e2..6fc0bdf110 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -10276,20 +10276,14 @@ int ObDDLResolver::resolve_partition_hash_or_key( LOG_WARN("failed to generate default hash part", K(ret)); } if (!stmt->use_def_sub_part()) { - if (table_schema.is_hash_like_subpart()) { - // partition by hash(c1) subpartition by hash(c2) subpartitions 3 partitions 3 - for (int64_t i = 0; OB_SUCC(ret) && i < table_schema.get_first_part_num(); ++i) { - if (OB_FAIL(resolve_subpartition_elements(stmt, - NULL, // dummy node - table_schema, - table_schema.get_part_array()[i], - false))) { - LOG_WARN("failed to resolve subpartition elements", K(ret)); - } + for (int64_t i = 0; OB_SUCC(ret) && i < table_schema.get_first_part_num(); ++i) { + if (OB_FAIL(resolve_subpartition_elements(stmt, + NULL, // dummy node + table_schema, + table_schema.get_part_array()[i], + false))) { + LOG_WARN("failed to resolve subpartition elements", K(ret)); } - } else { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid partition define", K(ret)); } } } @@ -11935,6 +11929,9 @@ int ObDDLResolver::parse_column_group(const ParseNode *column_group_node, if (OB_ISNULL(column_group_node)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("column gorup node should not be null", K(ret)); + } else if (!ObSchemaUtils::can_add_column_group(table_schema)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not supported table type to add column group", K(ret)); } else { dst_table_schema.set_max_used_column_group_id(table_schema.get_max_used_column_group_id()); } diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.h b/src/sql/resolver/ddl/ob_ddl_resolver.h index 65f28f8cd3..dab120bedc 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.h +++ b/src/sql/resolver/ddl/ob_ddl_resolver.h @@ -50,7 +50,7 @@ struct PartitionInfo enum NUMCHILD { CREATE_TABLE_NUM_CHILD = 8, - CREATE_TABLE_AS_SEL_NUM_CHILD = 9, + CREATE_TABLE_AS_SEL_NUM_CHILD = 10, COLUMN_DEFINITION_NUM_CHILD = 4, COLUMN_DEF_NUM_CHILD = 3, INDEX_NUM_CHILD = 5, diff --git a/src/sql/resolver/dml/ob_aggr_expr_push_up_analyzer.cpp b/src/sql/resolver/dml/ob_aggr_expr_push_up_analyzer.cpp index 79e4659cc0..70cda9b37a 100644 --- a/src/sql/resolver/dml/ob_aggr_expr_push_up_analyzer.cpp +++ b/src/sql/resolver/dml/ob_aggr_expr_push_up_analyzer.cpp @@ -37,7 +37,7 @@ int ObAggrExprPushUpAnalyzer::analyze_and_push_up_aggr_expr(ObRawExprFactory &ex if (OB_ISNULL(aggr_expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("aggr expr is null", K(ret)); - } else if (OB_FAIL(analyze_aggr_param_expr(root_expr, true))) { + } else if (OB_FAIL(analyze_aggr_param_expr(root_expr, false, true))) { LOG_WARN("failed to analyze aggr param expr", K(ret), K(*root_expr)); } else if (OB_FAIL(get_min_level_resolver(min_level_resolver))) { LOG_WARN("failed to get min level resolver", K(ret)); @@ -99,6 +99,7 @@ int ObAggrExprPushUpAnalyzer::analyze_and_push_up_aggr_expr(ObRawExprFactory &ex * @return */ int ObAggrExprPushUpAnalyzer::analyze_aggr_param_expr(ObRawExpr *¶m_expr, + bool is_in_aggr_expr, bool is_root /* = false*/, bool is_child_stmt /* = false*/) { @@ -130,7 +131,7 @@ int ObAggrExprPushUpAnalyzer::analyze_aggr_param_expr(ObRawExpr *¶m_expr, if (OB_SUCC(ret) && !is_child_stmt && param_expr->is_aggr_expr() && - !static_cast(param_expr)->is_nested_aggr()) { + !is_in_aggr_expr) { //在聚集函数中含有同层级的聚集函数，这个对于mysql不允许的 //select count(select count(t1.c1) from t) from t1; //在上面的例子中，count(t1.c1)推上去了，和最外层的count()处于同一级 @@ -143,7 +144,10 @@ int ObAggrExprPushUpAnalyzer::analyze_aggr_param_expr(ObRawExpr *¶m_expr, for (int64_t i = 0; OB_SUCC(ret) && i < param_expr->get_param_count(); ++i) { ObRawExpr *¶m = param_expr->get_param_expr(i); - if (OB_FAIL(SMART_CALL(analyze_aggr_param_expr(param, false, is_child_stmt)))) { + if (OB_ISNULL(param)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(SMART_CALL(analyze_aggr_param_expr(param, is_in_aggr_expr || param_expr->is_aggr_expr(), false, is_child_stmt)))) { LOG_WARN("analyze child expr failed", K(ret)); } } @@ -170,7 +174,10 @@ int ObAggrExprPushUpAnalyzer::analyze_child_stmt(ObSelectStmt *child_stmt) ObRawExpr *expr = NULL; if (OB_FAIL(relation_exprs.at(i).get(expr))) { LOG_WARN("failed to get expr", K(ret)); - } else if (OB_FAIL(analyze_aggr_param_expr(expr, false, true))) { + } else if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(analyze_aggr_param_expr(expr, false, false, true))) { LOG_WARN("failed to analyze aggr param expr", K(ret)); } else if (OB_FAIL(relation_exprs.at(i).set(expr))) { LOG_WARN("failed to set expr", K(ret)); diff --git a/src/sql/resolver/dml/ob_aggr_expr_push_up_analyzer.h b/src/sql/resolver/dml/ob_aggr_expr_push_up_analyzer.h index 38ec245f18..c56db108df 100644 --- a/src/sql/resolver/dml/ob_aggr_expr_push_up_analyzer.h +++ b/src/sql/resolver/dml/ob_aggr_expr_push_up_analyzer.h @@ -51,6 +51,7 @@ public: ObRawExpr *&final_aggr); private: int analyze_aggr_param_expr(ObRawExpr *¶m_expr, + bool is_in_aggr_expr, bool is_root = false, bool is_child_stmt = false); diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index 6e99db3c7f..ae360dc2e3 100755 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -4689,7 +4689,7 @@ int ObDMLResolver::resolve_joined_table_item(const ParseNode &parse_node, Joined table_node = parse_node.children_[i]; // nested join case or normal join case if (T_JOINED_TABLE == table_node->type_) { - if (OB_FAIL(resolve_joined_table(*table_node, child_table))) { + if (OB_FAIL(SMART_CALL(resolve_joined_table(*table_node, child_table)))) { LOG_WARN("resolve child joined table failed", K(ret)); } else if (1 == i) { cur_table->left_table_ = child_table; @@ -17431,6 +17431,8 @@ int ObDMLResolver::try_add_join_table_for_fts(const TableItem *left_table, Table ret = OB_ALLOCATE_MEMORY_FAILED; STORAGE_FTS_LOG(WARN, "fail to allocate right table item", K(ret)); } else { + ObRawExpr *part_expr = get_stmt()->get_part_expr(left_table->table_id_, left_table->ref_id_); + ObRawExpr *subpart_expr = get_stmt()->get_subpart_expr(left_table->table_id_, left_table->ref_id_); right_table->type_ = TableItem::BASE_TABLE; right_table->ref_id_ = table_schema->get_table_id(); right_table->table_id_ = generate_table_id(); @@ -17441,8 +17443,8 @@ int ObDMLResolver::try_add_join_table_for_fts(const TableItem *left_table, Table right_table->table_type_ = table_schema->get_table_type(); if (OB_FAIL(get_stmt()->add_table_item(session_info_, right_table))) { STORAGE_FTS_LOG(WARN, "fail to add right table item", K(ret), K(right_table)); - } else if (OB_FAIL(resolve_table_partition_expr(*right_table, *table_schema))) { - STORAGE_FTS_LOG(WARN, "fail to resolve table partition expr", K(ret), KPC(right_table), KPC(table_schema)); + } else if (OB_FAIL(get_stmt()->set_part_expr(right_table->table_id_, right_table->ref_id_, part_expr, subpart_expr))) { + STORAGE_FTS_LOG(WARN, "fail to set right table partition expr", K(ret), KPC(right_table), KPC(left_table)); } else if (OB_FAIL(create_joined_table_item(ObJoinType::INNER_JOIN, left_table, right_table, joined_table))) { STORAGE_FTS_LOG(WARN, "fail to create joined table item", K(ret), KPC(left_table), KPC(right_table)); } else if (OB_FAIL(add_all_rowkey_columns_to_stmt(*left_table, left_column_exprs))) { diff --git a/src/sql/resolver/dml/ob_group_by_checker.cpp b/src/sql/resolver/dml/ob_group_by_checker.cpp index a170884a69..c2d8b801b4 100644 --- a/src/sql/resolver/dml/ob_group_by_checker.cpp +++ b/src/sql/resolver/dml/ob_group_by_checker.cpp @@ -742,6 +742,8 @@ int ObGroupByChecker::check_select_stmt(const ObSelectStmt *ref_stmt) visitor.add_scope(SCOPE_ORDERBY); } else { // 如果是subquery，则需要check所有expression + // following is not allow, keep the old logic + // select (select d2 from t2 where max(t1.c1)=t2.d1) as c3 from t1 group by c1,c2; } // if order by has siblings, then order by belongs to connect by, so don't check // eg: select max(c2) from t1 start with c1 = 1 connect by nocycle prior c1 = c2 order siblings by c1, c2; @@ -915,6 +917,18 @@ int ObGroupByChecker::visit(ObMatchFunRawExpr &expr) return ret; } + +// following case is allowed +// select max(max(data)) from test group by id order by id, max(max(data)) ; +// select max(max(data)) from test group by id order by id; +// select max(max(data)) from test group by id having id=1; +// select sum(sum(data)) from test group by id order by sum(data); +// select sum(sum(data)) from test group by id having max(data) = 4 order by max(max(data)) +// following case is blocked +// select max(data) from test group by id order by max(max(data)) +// select max(data) from test group by id order by max(max(data)),max(data) +// having orderby select的顺序检查 +// having一定在内层，order by如果包含max max就在外层，否则在内层，select在外层 int ObGroupByChecker::visit(ObAggFunRawExpr &expr) { int ret = OB_SUCCESS; @@ -931,10 +945,10 @@ int ObGroupByChecker::visit(ObAggFunRawExpr &expr) LOG_WARN("failed to get belongs to stmt", K(ret)); } else if (belongs_to) { // expr is aggregate function in current stmt, then skip it - if (expr.is_nested_aggr()) { + if (expr.in_inner_stmt()) { set_skip_expr(&expr); } else if (has_nested_aggr_) { - //do nothing. + // do nothing } else { set_skip_expr(&expr); } diff --git a/src/sql/resolver/dml/ob_select_resolver.cpp b/src/sql/resolver/dml/ob_select_resolver.cpp index cfaf633f76..0c01c89384 100644 --- a/src/sql/resolver/dml/ob_select_resolver.cpp +++ b/src/sql/resolver/dml/ob_select_resolver.cpp @@ -1033,6 +1033,268 @@ int ObSelectResolver::search_connect_group_by_clause(const ParseNode &parent, return ret; } +int ObSelectResolver::check_and_mark_aggr_in_having_scope(ObSelectStmt *select_stmt) { + int ret = OB_SUCCESS; + if (OB_ISNULL(select_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null pointer", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < select_stmt->get_having_expr_size(); ++i) { + ObRawExpr* expr = select_stmt->get_having_exprs().at(i); + ObArray aggrs; + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr is NULL ptr", K(ret)); + } else if (ObTransformUtils::extract_aggr_expr(expr, aggrs)) { + LOG_WARN("failed to extrace aggr expr", K(ret)); + } else { + // having aggr must in inner stmt + for (int64_t j = 0; OB_SUCC(ret) && j < aggrs.count(); ++j) { + ObAggFunRawExpr* aggr_expr = aggrs.at(j); + if (OB_ISNULL(aggr_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr is NULL ptr", K(ret)); + } else if (aggr_expr->contain_nested_aggr()) { + ret = OB_ERR_WRONG_FIELD_WITH_GROUP; + LOG_USER_ERROR(OB_ERR_WRONG_FIELD_WITH_GROUP, + aggr_expr->get_expr_name().length(), + aggr_expr->get_expr_name().ptr()); + } else { + aggr_expr->set_nested_aggr_inner_stmt(true); + } + } + } + } + } + return ret; +} + +// block : +// 1 id and aggr(aggr(col)) in diff level +// select id from test group by id order by max(max(id)); +// select id from test group by id order by max(max(data)); +// select id, max(max(data)) from test group by id; +// select item must be outer +// select max(data) + 1 as data1 group by id order by max(data1); +int ObSelectResolver::check_aggr_in_select_scope(ObSelectStmt *select_stmt) { + int ret = OB_SUCCESS; + if (OB_ISNULL(select_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null pointer", K(ret)); + } else { + ObIArray &select_items = select_stmt->get_select_items(); + for (int64_t i = 0; OB_SUCC(ret) && i < select_items.count(); ++i) { + ObArray aggrs; + if (OB_ISNULL(select_items.at(i).expr_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid expr in select items.", K(ret)); + //compatible oracle: select 1, sum(max(c1)) from t1 group by c1; + } else if (select_items.at(i).expr_->is_const_expr()) { + //do nothing + } else if (!select_items.at(i).expr_->has_flag(CNT_AGG)) { + //in oracle it's "not a single-group group function." + // select id, max(max(id)) + ret = OB_ERR_WRONG_FIELD_WITH_GROUP; + ObString column_name = select_items.at(i).is_real_alias_ ? + select_items.at(i).alias_name_ : + select_items.at(i).expr_name_; + LOG_USER_ERROR(OB_ERR_WRONG_FIELD_WITH_GROUP, + column_name.length(), + column_name.ptr()); + } else if (OB_FAIL(ObTransformUtils::extract_aggr_expr(select_items.at(i).expr_, aggrs))){ + LOG_WARN("failed to extrace aggr_expr", K(ret)); + } else { + for (int64_t j = 0; OB_SUCC(ret) && j < aggrs.count(); ++j) { + ObAggFunRawExpr* aggr_expr = aggrs.at(j); + if (OB_ISNULL(aggr_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr is NULL ptr", K(ret)); + } else if (aggr_expr->in_inner_stmt()) { + ret = OB_ERR_NOT_A_SINGLE_GROUP_FUNCTION; + LOG_WARN("select in aggr alias can not be nested in aggr", K(ret)); + } + } + } + } + if (OB_SUCC(ret)) { + if (select_stmt->get_group_expr_size() == 0 && + select_stmt->get_rollup_expr_size() == 0 && + select_stmt->get_grouping_sets_items_size() == 0 && + select_stmt->get_rollup_items_size() == 0 && + select_stmt->get_cube_items_size() == 0) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("nested group function without group by", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "nested group function without group by"); + } + } + } + return ret; +} + +int ObSelectResolver::mark_aggr_in_select_scope(ObSelectStmt *select_stmt) { + int ret = OB_SUCCESS; + if (OB_ISNULL(select_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null pointer", K(ret)); + } else { + ObRawExprCopier copier(*params_.expr_factory_); + ObIArray &select_items = select_stmt->get_select_items(); + ObSEArray origin_mark_inner_expr; + for (int64_t i = 0; OB_SUCC(ret) && i < select_items.count(); ++i) { + ObArray aggrs; + if (OB_ISNULL(select_items.at(i).expr_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid expr in select items.", K(ret)); + } else if (OB_FAIL(ObTransformUtils::extract_aggr_expr(select_items.at(i).expr_, aggrs))){ + LOG_WARN("failed to extrace aggr_eObIRawExprCopierxpr", K(ret)); + } else { + for (int64_t j = 0; OB_SUCC(ret) && j < aggrs.count(); ++j) { + ObAggFunRawExpr* aggr_expr = aggrs.at(j); + if (OB_ISNULL(aggr_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr is NULL ptr", K(ret)); + } else if (aggr_expr->in_inner_stmt()) { + // select max(id) + 1 from test group by id having max(id) = 1 order by max(id),max(max(data)); + // select sum(b),sum(b) + sum(c) as inn from t3 group by b,c having sum(b)+sum(c) > 1 order by 1,sum(b) + sum(sum(e + c)); + if(OB_FAIL(add_var_to_array_no_dup(origin_mark_inner_expr, aggr_expr))) { + LOG_WARN("error to add aggr to array", K(ret)); + } + } else { + aggr_expr->set_nested_aggr_inner_stmt(false); + } + } + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < origin_mark_inner_expr.count(); ++i) { + ObRawExpr *aggr_expr = origin_mark_inner_expr.at(i); + ObRawExpr *aggr_expr_cp = NULL; + if (OB_FAIL(ObRawExprCopier::copy_expr_node(*params_.expr_factory_, aggr_expr, aggr_expr_cp))) { + LOG_WARN("failed to expr copy", K(ret)); + } else if (aggr_expr_cp == NULL) { + LOG_WARN("unexpected null ptr", K(ret)); + } else { + ObAggFunRawExpr* new_agg = static_cast(aggr_expr_cp); + new_agg->set_nested_aggr_inner_stmt(false); + if (OB_FAIL(copier.add_replaced_expr(aggr_expr, new_agg))) { + LOG_WARN("failed to add replace expr", K(ret)); + } else if (OB_FAIL(select_stmt->add_agg_item(*new_agg))) { + LOG_WARN("failed to add agg item", K(ret)); + } + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < select_items.count(); ++i) { + ObRawExpr* new_expr = NULL; + if (OB_ISNULL(select_items.at(i).expr_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid expr in select items.", K(ret)); + } else if (OB_FAIL(copier.copy_on_replace(select_items.at(i).expr_, new_expr))) { + LOG_WARN("failed to copy on replace the expr", K(ret)); + } else if (new_expr == NULL) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null ptr", K(ret)); + } else { + select_items.at(i).expr_ = new_expr; + } + } + ObIArray &order_items = select_stmt->get_order_items(); + for (int64_t i = 0; OB_SUCC(ret) && i < order_items.count(); ++i) { + ObRawExpr* new_expr = NULL; + if (OB_ISNULL(order_items.at(i).expr_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid expr in order items.", K(ret)); + } else if (OB_FAIL(copier.copy_on_replace(order_items.at(i).expr_, new_expr))) { + LOG_WARN("failed to copy on replace the expr", K(ret)); + } else if (new_expr == NULL) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null ptr", K(ret)); + } else { + order_items.at(i).expr_ = new_expr; + } + } + } + return ret; +} + +// if orderby has aggr(aggr) then orderby should be outer else it should be inner +// positive example following max(data) have to checked; +// select max(data) group by id order by max(max(data)); +// select max(id) group by id order by max(max(id)); +// negetive example following max(data) should not to be checked in inner stmt +// select id from test group by id having max(data) = 1 ordered by max(max(data)) +int ObSelectResolver::mark_aggr_in_order_by_scope(ObSelectStmt *select_stmt) { + int ret = OB_SUCCESS; + if (OB_ISNULL(select_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null pointer", K(ret)); + } else { + ObIArray &select_items = select_stmt->get_select_items(); + ObSEArray select_agg_expr; + + for (int64_t i = 0; OB_SUCC(ret) && i < select_items.count(); ++i) { + ObArray aggrs; + if (OB_ISNULL(select_items.at(i).expr_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid expr in select items.", K(ret)); + } else if (OB_FAIL(ObTransformUtils::extract_aggr_expr(select_items.at(i).expr_, aggrs))){ + LOG_WARN("failed to extrace aggr_eObIRawExprCopierxpr", K(ret)); + } else { + for (int64_t j = 0; OB_SUCC(ret) && j < aggrs.count(); ++j) { + ObAggFunRawExpr* aggr_expr = aggrs.at(j); + if (OB_ISNULL(aggr_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null ptr", K(ret)); + } else if (OB_FAIL(add_var_to_array_no_dup(select_agg_expr, aggr_expr))) { + LOG_WARN("error to add aggr to array", K(ret)); + } + } + } + } + + ObIArray &order_items = select_stmt->get_order_items(); + for (int64_t i = 0; OB_SUCC(ret) && i < order_items.count(); ++i) { + ObArray aggrs; + if (OB_ISNULL(order_items.at(i).expr_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid expr in select items.", K(ret)); + //compatible oracle: select 1, sum(max(c1)) from t1 group by c1; + } else if (order_items.at(i).expr_->is_const_expr()) { + // do nothing + } else if (OB_FAIL(ObTransformUtils::extract_aggr_expr(order_items.at(i).expr_, aggrs))) { + LOG_WARN("invalid expr in extrace aggr expr", K(ret)); + } else { + for (int64_t j = 0; OB_SUCC(ret) && j < aggrs.count(); ++j) { + ObAggFunRawExpr* aggr_expr = aggrs.at(j); + if (OB_ISNULL(aggr_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr is NULL ptr", K(ret)); + } else if (aggr_expr->contain_nested_aggr()) { + aggr_expr->set_nested_aggr_inner_stmt(false); + } else { + if(!aggr_expr->in_inner_stmt()) { + // there are two types of aggr in order by + // 1 derive from the select -- outer + // 2 derive from the having -- inner + // 3 new appear in order by -- inner + // select max(id) from test group by id order by 1,max(max(data)); + // select max(id) from test group by id order by max(id),max(max(data)); + // select sum(b) + sum(c) from t3 group by b,c having sum(b)+sum(c) > 1 order by 1,sum(b) + sum(sum(e + c)); + // select sum(b) + sum(c),sum(sum(b)) from t3 group by b,c having sum(b)+sum(c) > 1 order by 1,sum(b) + sum(sum(e + c)); + // In oracle next stmt can be compiled but running with error. + // select sum(b) + sum(c) from t3 group by b,c having sum(b)+sum(c) > 1 order by 1,sum(b) + sum(sum(e + c)) + sum(e); + if (!has_exist_in_array(select_agg_expr, aggr_expr)) { + aggr_expr->set_nested_aggr_inner_stmt(true); + } + } + // this branch means aggr_expr in_inner_stmt + // select sub(c) from test group by b having sum(b) > 1 order by sum(b) + } + } + } + } + } + return ret; +} + int ObSelectResolver::resolve_normal_query(const ParseNode &parse_tree) { int ret = OB_SUCCESS; @@ -1167,6 +1429,18 @@ int ObSelectResolver::resolve_normal_query(const ParseNode &parse_tree) OZ( select_stmt->formalize_stmt(session_info_) ); + if (OB_SUCC(ret) && has_nested_aggr_) { + if (OB_FAIL(check_aggr_in_select_scope(select_stmt))) { + LOG_WARN("failed to check expr in select scope", K(ret)); + } else if (OB_FAIL(check_and_mark_aggr_in_having_scope(select_stmt))) { + LOG_WARN("failed to check and mark the expr having", K(ret)); + } else if (OB_FAIL(mark_aggr_in_select_scope(select_stmt))) { + LOG_WARN("failed to check and mark the expr select", K(ret)); + } else if (OB_FAIL(mark_aggr_in_order_by_scope(select_stmt))) { + LOG_WARN("failed to check and mark the expr order", K(ret)); + } + } + //统一为本层的表达式进行only full group by验证，避免检查的逻辑过于分散 OZ( check_group_by() ); OZ( check_order_by() ); @@ -2220,44 +2494,14 @@ int ObSelectResolver::resolve_field_list(const ParseNode &node) // for oracle mode, check grouping here if (OB_FAIL(ret) || !is_oracle_mode()) { /*do nothing*/ - } else if (OB_FAIL(recursive_check_grouping_columns(select_stmt, select_item.expr_))) { + } else if (OB_ISNULL(select_item.expr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(recursive_check_grouping_columns(select_stmt, select_item.expr_, false))) { LOG_WARN("failed to recursive check grouping columns", K(ret)); } else {/*do nothing*/} } // end for - if (OB_SUCC(ret) && has_nested_aggr_) { - ObIArray &select_items = select_stmt->get_select_items(); - for (int64_t i = 0; OB_SUCC(ret) && i < select_items.count(); i++) { - if (OB_ISNULL(select_items.at(i).expr_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid expr in select items.", K(ret)); - //compatible oracle: select 1, sum(max(c1)) from t1 group by c1; - } else if (select_items.at(i).expr_->is_const_expr()) { - //do nothing - } else if (!select_items.at(i).expr_->has_flag(CNT_AGG)) { - //in oracle it's "not a single-group group function." - ret = OB_ERR_WRONG_FIELD_WITH_GROUP; - ObString column_name = select_items.at(i).is_real_alias_ ? - select_items.at(i).alias_name_ : - select_items.at(i).expr_name_; - LOG_USER_ERROR(OB_ERR_WRONG_FIELD_WITH_GROUP, - column_name.length(), - column_name.ptr()); - } else { /*do nothing.*/ } - } - if (OB_SUCC(ret)) { - if (select_stmt->get_group_expr_size() == 0 && - select_stmt->get_rollup_expr_size() == 0 && - select_stmt->get_grouping_sets_items_size() == 0 && - select_stmt->get_rollup_items_size() == 0 && - select_stmt->get_cube_items_size() == 0) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("nested group function without group by", K(ret)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "nested group function without group by"); - } - } - } - if (OB_SUCC(ret)) { // Oracle mode, * can't be used with other expresion // like: select 1,* from t1; select *,1 from t1; @@ -4417,7 +4661,10 @@ int ObSelectResolver::check_grouping_columns() } else { common::ObIArray &select_items = select_stmt->get_select_items(); for (int64_t i = 0; OB_SUCC(ret) && i < select_items.count(); ++i) { - if (OB_FAIL(recursive_check_grouping_columns(select_stmt, select_items.at(i).expr_))) { + if (OB_ISNULL(select_items.at(i).expr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(recursive_check_grouping_columns(select_stmt, select_items.at(i).expr_, false))) { LOG_WARN("failed to recursive check grouping columns", K(ret)); } } @@ -4490,39 +4737,6 @@ int ObSelectResolver::check_grouping_columns(ObSelectStmt &stmt, ObRawExpr *&exp return ret; } -int ObSelectResolver::check_nested_aggr_in_having(ObRawExpr* raw_expr) -{ - int ret = OB_SUCCESS; - bool is_stack_overflow = false; - if (OB_FAIL(check_stack_overflow(is_stack_overflow))) { - LOG_WARN("check stack overflow failed", K(ret)); - } else if (is_stack_overflow) { - ret = OB_SIZE_OVERFLOW; - LOG_WARN("stack is overflow", K(ret), K(is_stack_overflow)); - } else if (OB_ISNULL(raw_expr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("raw expr is NULL ptr", K(ret)); - } else { - int64_t N = raw_expr->get_param_count(); - for (int64_t i = 0; OB_SUCC(ret) && i < N; ++i) { - ObRawExpr *child_expr = raw_expr->get_param_expr(i); - if (OB_ISNULL(child_expr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("expr is NULL ptr", K(ret)); - } else if (child_expr->is_aggr_expr() && - static_cast(child_expr)->is_nested_aggr()) { - ret = OB_ERR_WRONG_FIELD_WITH_GROUP; - LOG_USER_ERROR(OB_ERR_WRONG_FIELD_WITH_GROUP, - child_expr->get_expr_name().length(), - child_expr->get_expr_name().ptr()); - } else if (OB_FAIL(SMART_CALL(check_nested_aggr_in_having(child_expr)))) { - LOG_WARN("replace reference column failed", K(ret)); - } else { /*do nothing.*/ } - } // end for - } - return ret; -} - int ObSelectResolver::resolve_having_clause(const ParseNode *node) { int ret = OB_SUCCESS; @@ -4538,9 +4752,7 @@ int ObSelectResolver::resolve_having_clause(const ParseNode *node) if (OB_ISNULL(expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("expr is NULL ptr", K(ret)); - } else if (OB_FAIL(check_nested_aggr_in_having(expr))) { - LOG_WARN("failed to check nested aggr in having.", K(ret)); - } else if (OB_FAIL(recursive_check_grouping_columns(select_stmt, expr))) { + } else if (OB_FAIL(recursive_check_grouping_columns(select_stmt, expr, false))) { LOG_WARN("failed to recursive check grouping columns", K(ret)); } else if (expr->has_flag(CNT_ROWNUM) || expr->has_flag(CNT_LEVEL) @@ -5692,7 +5904,7 @@ int ObSelectResolver::mark_nested_aggr_if_required( } } } - if (has_nested_aggr_) { + if (OB_SUCC(ret) && has_nested_aggr_) { ObArray param_aggrs; ObArray param_winfuncs; if (OB_UNLIKELY(is_mysql_mode())) { @@ -5736,7 +5948,7 @@ int ObSelectResolver::mark_nested_aggr_if_required( ret = OB_ERR_INVALID_GROUP_FUNC_USE; LOG_WARN("nested aggr should not contain nested aggr", K(ret)); } else { - param_aggrs.at(i)->set_in_nested_aggr(true); + param_aggrs.at(i)->set_nested_aggr_inner_stmt(true); } } } @@ -5818,7 +6030,7 @@ int ObSelectResolver::add_aggr_expr(ObAggFunRawExpr *&final_aggr_expr) } else if (OB_UNLIKELY(select_stmt->is_set_stmt())) { ret = OB_ERR_AGGREGATE_ORDER_FOR_UNION; LOG_WARN("can't use aggregate function in union stmt"); - } else if (OB_FAIL(select_stmt->check_and_get_same_aggr_item(final_aggr_expr, + } else if (OB_FAIL(select_stmt->check_and_get_same_aggr_item(final_aggr_expr, // 这里实际上判断是错误的 same_aggr_expr))) { LOG_WARN("failed to check and get same aggr item.", K(ret)); } else if (same_aggr_expr != NULL) { @@ -6849,7 +7061,7 @@ int ObSelectResolver::check_cube_items_valid(const ObIArray &cube_it return ret; } -int ObSelectResolver::recursive_check_grouping_columns(ObSelectStmt *stmt, ObRawExpr *expr) +int ObSelectResolver::recursive_check_grouping_columns(ObSelectStmt *stmt, ObRawExpr *expr, bool is_in_aggr) { int ret = OB_SUCCESS; ObAggFunRawExpr *c_expr = NULL; @@ -6862,7 +7074,7 @@ int ObSelectResolver::recursive_check_grouping_columns(ObSelectStmt *stmt, ObRaw if (OB_ISNULL(c_expr = static_cast(expr))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unable to convert expr to ObAggFunRawExpr", K(ret)); - } else if (c_expr->is_nested_aggr()) { + } else if (is_in_aggr && c_expr->is_aggr_expr()) { ret = OB_ERR_GROUP_FUNC_NOT_ALLOWED; LOG_WARN("grouping shouldn't be nested", K(ret)); } else if (1 != c_expr->get_real_param_exprs().count() || @@ -6879,7 +7091,7 @@ int ObSelectResolver::recursive_check_grouping_columns(ObSelectStmt *stmt, ObRaw if (OB_ISNULL(c_expr = static_cast(expr))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unable to convert expr to ObAggFunRawExpr", K(ret)); - } else if (c_expr->is_nested_aggr()) { + } else if (is_in_aggr && c_expr->is_aggr_expr()) { ret = OB_ERR_GROUP_FUNC_NOT_ALLOWED; LOG_WARN("grouping shouldn't be nested", K(ret)); } else if (c_expr->get_real_param_count() < 1) { @@ -6897,7 +7109,7 @@ int ObSelectResolver::recursive_check_grouping_columns(ObSelectStmt *stmt, ObRaw if (OB_ISNULL(c_expr = static_cast(expr))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unable to convert expr to ObAggFunRawExpr", K(ret)); - } else if (c_expr->is_nested_aggr()) { + } else if (is_in_aggr && c_expr->is_aggr_expr()) { ret = OB_ERR_GROUP_FUNC_NOT_ALLOWED; LOG_WARN("group_id shouldn't be nested", K(ret)); } else if (stmt->get_group_expr_size() == 0 && @@ -6910,7 +7122,11 @@ int ObSelectResolver::recursive_check_grouping_columns(ObSelectStmt *stmt, ObRaw } } else { for (int64_t i = 0; OB_SUCC(ret) && i < expr->get_param_count(); ++i) { - if (OB_FAIL(SMART_CALL(recursive_check_grouping_columns(stmt, expr->get_param_expr(i))))) { + ObRawExpr *expr_param = expr->get_param_expr(i); + if (OB_ISNULL(expr_param)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(SMART_CALL(recursive_check_grouping_columns(stmt, expr_param, is_in_aggr || expr->is_aggr_expr())))) { LOG_WARN("failed to recursive check grouping columns", K(ret)); } else {/*do nothing*/} } diff --git a/src/sql/resolver/dml/ob_select_resolver.h b/src/sql/resolver/dml/ob_select_resolver.h index 46f9a984f9..a4a01ef8c2 100644 --- a/src/sql/resolver/dml/ob_select_resolver.h +++ b/src/sql/resolver/dml/ob_select_resolver.h @@ -221,7 +221,6 @@ protected: ObRawExpr *&coalesce_expr); int resolve_having_clause(const ParseNode *node); int resolve_named_windows_clause(const ParseNode *node); - int check_nested_aggr_in_having(ObRawExpr* expr); int resolve_start_with_clause(const ParseNode *node); int check_connect_by_expr_validity(const ObRawExpr *raw_expr, bool is_prior); int resolve_connect_by_clause(const ParseNode *node); @@ -347,7 +346,7 @@ private: int check_rollup_items_valid(const common::ObIArray &rollup_items); int check_cube_items_valid(const common::ObIArray &cube_items); - int recursive_check_grouping_columns(ObSelectStmt *stmt, ObRawExpr *expr); + int recursive_check_grouping_columns(ObSelectStmt *stmt, ObRawExpr *expr, bool is_in_aggr_expr); int is_need_check_col_dup(const ObRawExpr *expr, bool &need_check); @@ -358,6 +357,12 @@ private: int check_listagg_aggr_param_valid(ObAggFunRawExpr *aggr_expr); int add_alias_from_dot_notation(ObRawExpr *sel_expr, SelectItem& select_item); + + int check_and_mark_aggr_in_having_scope(ObSelectStmt *select_stmt); + int mark_aggr_in_order_by_scope(ObSelectStmt *select_stmt); + int check_aggr_in_select_scope(ObSelectStmt *select_stmt); + int mark_aggr_in_select_scope(ObSelectStmt *select_stmt); + protected: // data members /*these member is only for with clause*/ diff --git a/src/sql/resolver/dml/ob_select_stmt.cpp b/src/sql/resolver/dml/ob_select_stmt.cpp index b25afd0288..37292ca164 100644 --- a/src/sql/resolver/dml/ob_select_stmt.cpp +++ b/src/sql/resolver/dml/ob_select_stmt.cpp @@ -156,7 +156,7 @@ int ObSelectStmt::check_aggr_and_winfunc(ObRawExpr &expr) if (expr.is_aggr_expr() && !ObRawExprUtils::find_expr(agg_items_, &expr)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("aggr expr does not exist in the stmt", K(ret), K(expr)); + LOG_WARN("aggr expr does not exist in the stmt", K(agg_items_), K(expr), K(ret)); } else if (expr.is_win_func_expr() && !ObRawExprUtils::find_expr(win_func_exprs_, &expr)) { ret = OB_ERR_UNEXPECTED; diff --git a/src/sql/resolver/expr/ob_raw_expr.cpp b/src/sql/resolver/expr/ob_raw_expr.cpp index 32bdd98ea1..3d987f91e8 100644 --- a/src/sql/resolver/expr/ob_raw_expr.cpp +++ b/src/sql/resolver/expr/ob_raw_expr.cpp @@ -3679,7 +3679,7 @@ int ObAggFunRawExpr::assign(const ObRawExpr &other) } else { distinct_ = tmp.distinct_; separator_param_expr_ = tmp.separator_param_expr_; - is_nested_aggr_ = tmp.is_nested_aggr_; + expr_in_inner_stmt_ = tmp.expr_in_inner_stmt_; pl_agg_udf_expr_ = tmp.pl_agg_udf_expr_; udf_meta_.assign(tmp.udf_meta_); is_need_deserialize_row_ = tmp.is_need_deserialize_row_; @@ -3743,7 +3743,7 @@ void ObAggFunRawExpr::reset() clear_child(); order_items_.reset(); separator_param_expr_ = NULL; - is_nested_aggr_ = false; + expr_in_inner_stmt_ = false; is_need_deserialize_row_ = false; pl_agg_udf_expr_ = NULL; } @@ -3756,7 +3756,7 @@ bool ObAggFunRawExpr::inner_same_as( if (get_expr_type() != expr.get_expr_type()) { } else { const ObAggFunRawExpr *a_expr = static_cast(&expr); - if (is_nested_aggr_ != a_expr->is_nested_aggr()) { + if (expr_in_inner_stmt_ != a_expr->expr_in_inner_stmt_) { //do nothing. } else if (distinct_ == a_expr->is_param_distinct()) { if ((NULL == separator_param_expr_ && NULL == a_expr->separator_param_expr_) diff --git a/src/sql/resolver/expr/ob_raw_expr.h b/src/sql/resolver/expr/ob_raw_expr.h index 7fc22822cb..b6232de502 100644 --- a/src/sql/resolver/expr/ob_raw_expr.h +++ b/src/sql/resolver/expr/ob_raw_expr.h @@ -3372,7 +3372,7 @@ public: order_items_(), separator_param_expr_(NULL), udf_meta_(), - is_nested_aggr_(false), + expr_in_inner_stmt_(false), is_need_deserialize_row_(false), pl_agg_udf_expr_(NULL) { @@ -3386,7 +3386,7 @@ public: order_items_(), separator_param_expr_(NULL), udf_meta_(), - is_nested_aggr_(false), + expr_in_inner_stmt_(false), is_need_deserialize_row_(false), pl_agg_udf_expr_(NULL) { @@ -3401,7 +3401,7 @@ public: order_items_(), separator_param_expr_(NULL), udf_meta_(), - is_nested_aggr_(false), + expr_in_inner_stmt_(false), is_need_deserialize_row_(false), pl_agg_udf_expr_(NULL) { @@ -3420,8 +3420,8 @@ public: bool is_param_distinct() const; void set_param_distinct(bool is_distinct); void set_separator_param_expr(ObRawExpr *separator_param_expr); - bool is_nested_aggr() const; - void set_in_nested_aggr(bool is_nested); + bool in_inner_stmt() const; + void set_nested_aggr_inner_stmt(bool inner_stmt); int add_order_item(const OrderItem &order_item); virtual void clear_child() override; virtual void reset(); @@ -3479,8 +3479,9 @@ public: N_ORDER_BY, order_items_, N_SEPARATOR_PARAM_EXPR, separator_param_expr_, K_(udf_meta), - K_(is_nested_aggr), + K_(expr_in_inner_stmt), K_(pl_agg_udf_expr)); + private: DISALLOW_COPY_AND_ASSIGN(ObAggFunRawExpr); // real_param_exprs_.count() == 0 means '*' @@ -3491,7 +3492,7 @@ private: ObRawExpr *separator_param_expr_; //use for udf function info share::schema::ObUDFMeta udf_meta_; - bool is_nested_aggr_; + bool expr_in_inner_stmt_; bool is_need_deserialize_row_;// for topk histogram and hybrid histogram computation ObRawExpr *pl_agg_udf_expr_;//for pl agg udf expr }; @@ -3542,9 +3543,9 @@ inline bool ObAggFunRawExpr::is_param_distinct() const { return distinct_; } -inline bool ObAggFunRawExpr::is_nested_aggr() const +inline bool ObAggFunRawExpr::in_inner_stmt() const { - return is_nested_aggr_; + return expr_in_inner_stmt_; } inline bool ObAggFunRawExpr::contain_nested_aggr() const { @@ -3557,7 +3558,7 @@ inline bool ObAggFunRawExpr::contain_nested_aggr() const return ret; } inline void ObAggFunRawExpr::set_param_distinct(bool is_distinct) { distinct_ = is_distinct; } -inline void ObAggFunRawExpr::set_in_nested_aggr(bool is_nested) { is_nested_aggr_ = is_nested; } +inline void ObAggFunRawExpr::set_nested_aggr_inner_stmt(bool inner_stmt) { expr_in_inner_stmt_ = inner_stmt; } inline void ObAggFunRawExpr::set_separator_param_expr(ObRawExpr *separator_param_expr) { separator_param_expr_ = separator_param_expr; diff --git a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp index 6cdcd3c226..f0e1cfbd9c 100644 --- a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp @@ -2195,11 +2195,34 @@ int ObRawExprResolverImpl::check_pl_variable(ObQualifiedName &q_name, bool &is_p if (NULL == ctx_.secondary_namespace_) { // do nothing ... } else { + +class ObPLDependencyGuard +{ +public: + ObPLDependencyGuard(pl::ObPLDependencyTable *dependency_table) + : dependency_table_(dependency_table), count_(0) { + if (OB_NOT_NULL(dependency_table_)) { + count_ = dependency_table_->count(); + } + } + ~ObPLDependencyGuard() { + if (OB_NOT_NULL(dependency_table_)) { + while (dependency_table_->count() > count_) { + dependency_table_->pop_back(); + } + } + } +private: + pl::ObPLDependencyTable *dependency_table_; + int64_t count_; +}; + SET_LOG_CHECK_MODE(); CK(OB_NOT_NULL(ctx_.secondary_namespace_->get_external_ns())); if (OB_SUCC(ret)) { ObArray fake_columns; ObArray fake_exprs; + ObPLDependencyGuard dep_guard(ctx_.secondary_namespace_->get_external_ns()->get_dependency_table()); if (OB_FAIL(ObResolverUtils::resolve_external_symbol(allocator, expr_factory, ctx_.secondary_namespace_->get_external_ns()->get_resolve_ctx().session_info_, diff --git a/src/sql/resolver/mv/ob_mv_checker.cpp b/src/sql/resolver/mv/ob_mv_checker.cpp index 239770a61e..0548876352 100644 --- a/src/sql/resolver/mv/ob_mv_checker.cpp +++ b/src/sql/resolver/mv/ob_mv_checker.cpp @@ -459,7 +459,7 @@ int ObMVChecker::check_and_expand_mav_aggr(const ObSelectStmt &stmt, if (OB_ISNULL(aggr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret), K(aggr)); - } else if (aggr->is_param_distinct() || aggr->is_nested_aggr()) { + } else if (aggr->is_param_distinct() || aggr->in_inner_stmt()) { // 这里判断完全没有has_nested_aggr_的时候in_inner_stmt 才会为true is_valid = false; append_fast_refreshable_note("nested aggr and aggr with distinct param not support", OB_MV_FAST_REFRESH_SIMPLE_MAV); } else { diff --git a/src/sql/resolver/ob_resolver_utils.cpp b/src/sql/resolver/ob_resolver_utils.cpp index 5a6fed0d82..ec281d88bd 100644 --- a/src/sql/resolver/ob_resolver_utils.cpp +++ b/src/sql/resolver/ob_resolver_utils.cpp @@ -7409,6 +7409,7 @@ int ObResolverUtils::resolve_external_symbol(common::ObIAllocator &allocator, } } } + if (OB_SUCC(ret)) { pl::ObPLResolver pl_resolver(allocator, session_info, @@ -7446,7 +7447,7 @@ int ObResolverUtils::resolve_external_symbol(common::ObIAllocator &allocator, && T_FUN_PL_GET_CURSOR_ATTR != expr->get_expr_type()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("expr type is invalid", K(expr->get_expr_type())); - } else if (OB_NOT_NULL(ns) && OB_NOT_NULL(ns->get_external_ns())) { + } else if (OB_NOT_NULL(ns) && OB_NOT_NULL(ns->get_external_ns()) && !is_check_mode) { ObPLDependencyTable &src_dep_tbl = func_ast.get_dependency_table(); for (int64_t i = 0; OB_SUCC(ret) && i < src_dep_tbl.count(); ++i) { OZ (ns->get_external_ns()->add_dependency_object(src_dep_tbl.at(i))); diff --git a/src/sql/resolver/ob_stmt_type.h b/src/sql/resolver/ob_stmt_type.h index 9326ff2adc..bc9c157d82 100644 --- a/src/sql/resolver/ob_stmt_type.h +++ b/src/sql/resolver/ob_stmt_type.h @@ -340,6 +340,9 @@ OB_STMT_TYPE_DEF_UNKNOWN_AT(T_KV_MULTI_PUT, err_stmt_type_priv, 339) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_KV_QUERY, err_stmt_type_priv, 340) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_XA_RECOVER, no_priv_needed, 341) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_ADMIN_STORAGE, get_sys_tenant_alter_system_priv, 342) +OB_STMT_TYPE_DEF_UNKNOWN_AT(T_EVENT_JOB_CREATE, no_priv_needed, 343) +OB_STMT_TYPE_DEF_UNKNOWN_AT(T_EVENT_JOB_ALTER, no_priv_needed, 344) +OB_STMT_TYPE_DEF_UNKNOWN_AT(T_EVENT_JOB_DROP, no_priv_needed, 345) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_MAX, err_stmt_type_priv, 500) #endif diff --git a/src/sql/rewrite/ob_expand_aggregate_utils.cpp b/src/sql/rewrite/ob_expand_aggregate_utils.cpp index 215ba46e6a..b5b31b7492 100644 --- a/src/sql/rewrite/ob_expand_aggregate_utils.cpp +++ b/src/sql/rewrite/ob_expand_aggregate_utils.cpp @@ -2101,8 +2101,11 @@ int ObExpandAggregateUtils::expand_stddev_samp_expr(ObAggFunRawExpr *aggr_expr, LOG_WARN("get unexpected null", K(ret), K(aggr_expr)); } else { ObSysFunRawExpr *sqrt_expr = NULL; - ObRawExpr *sqrt_param_expr = NULL; + ObRawExpr *expand_var_expr_inner = NULL; + ObRawExpr *case_when_expr = NULL; + ObConstRawExpr *zero_expr = NULL; ObAggFunRawExpr *var_expr = NULL; + ObRawExpr *lt_expr = NULL; if (OB_FAIL(ObRawExprUtils::build_common_aggr_expr(expr_factory_, session_info_, T_FUN_VAR_SAMP, @@ -2111,19 +2114,49 @@ int ObExpandAggregateUtils::expand_stddev_samp_expr(ObAggFunRawExpr *aggr_expr, LOG_WARN("failed to build common aggr expr", K(ret)); } else { if (OB_FAIL(expand_var_expr(var_expr, - sqrt_param_expr, new_aggr_items))) { + expand_var_expr_inner, new_aggr_items))) { LOG_WARN("fail to expand keep variance expr", K(ret)); } else if (OB_FAIL(expr_factory_.create_raw_expr(T_FUN_SYS_SQRT, sqrt_expr))) { LOG_WARN("failed to crate sqrt expr", K(ret)); } else if (OB_ISNULL(sqrt_expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("add expr is null", K(ret), K(sqrt_expr)); - } else if (OB_FAIL(sqrt_expr->add_param_expr(sqrt_param_expr))) { - LOG_WARN("add text expr to add expr failed", K(ret)); } else { - ObString func_name = ObString::make_string("sqrt"); - sqrt_expr->set_func_name(func_name); - replace_expr = sqrt_expr; + if (is_oracle_mode()) { + if (OB_SUCC(ret)) { + if (OB_FAIL(sqrt_expr->add_param_expr(expand_var_expr_inner))) { + LOG_WARN("add text expr to add expr failed", K(ret)); + } + } + } else { + if (OB_SUCC(ret)) { + if (OB_FAIL(ObRawExprUtils::build_const_int_expr(expr_factory_, + ObIntType, + 0, + zero_expr))) { + LOG_WARN("failed to build const int expr", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::build_common_binary_op_expr(expr_factory_, + T_OP_LT, + expand_var_expr_inner, + zero_expr, + lt_expr))) { + LOG_WARN("failed to build common binary op expr", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::build_case_when_expr(expr_factory_, + lt_expr, + zero_expr, + expand_var_expr_inner, + case_when_expr))) { + LOG_WARN("failed to build the case when expr", K(ret)); + } else if (OB_FAIL(sqrt_expr->add_param_expr(case_when_expr))) { + LOG_WARN("add text expr to add expr failed", K(ret)); + } + } + } + if (OB_SUCC(ret)) { + ObString func_name = ObString::make_string("sqrt"); + sqrt_expr->set_func_name(func_name); + replace_expr = sqrt_expr; + } } } } diff --git a/src/sql/rewrite/ob_transform_pre_process.cpp b/src/sql/rewrite/ob_transform_pre_process.cpp index 3a10e163d9..2986e55f0b 100644 --- a/src/sql/rewrite/ob_transform_pre_process.cpp +++ b/src/sql/rewrite/ob_transform_pre_process.cpp @@ -4202,6 +4202,32 @@ int ObTransformPreProcess::replace_expr_for_rls(ObDMLStmt &stmt, return ret; } +int ObTransformPreProcess::get_inner_aggr_exprs( + ObSelectStmt *sub_stmt, + common::ObIArray& inner_aggr_exprs) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(sub_stmt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("select stmt is null", K(ret)); + } else { + common::ObIArray &aggr_exprs = sub_stmt->get_aggr_items(); + for (int64_t i = 0; OB_SUCC(ret) && i < aggr_exprs.count(); ++i) { + if (OB_ISNULL(aggr_exprs.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr of aggr expr is null", K(ret)); + } else if (aggr_exprs.at(i)->in_inner_stmt()) { + /*do nothing.*/ + if (OB_FAIL(add_var_to_array_no_dup(inner_aggr_exprs, + aggr_exprs.at(i)))) { + LOG_WARN("failed to to add var to array no dup.", K(ret)); + } + } + } + } + return ret; +} + int ObTransformPreProcess::get_first_level_output_exprs( ObSelectStmt *sub_stmt, common::ObIArray& inner_aggr_exprs) @@ -4212,13 +4238,14 @@ int ObTransformPreProcess::get_first_level_output_exprs( LOG_WARN("select stmt is null", K(ret)); } else { common::ObIArray &aggr_exprs = sub_stmt->get_aggr_items(); - for (int64_t i = 0; OB_SUCC(ret) && i < aggr_exprs.count(); i++) { + for (int64_t i = 0; OB_SUCC(ret) && i < aggr_exprs.count(); ++i) { if (OB_ISNULL(aggr_exprs.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("expr of aggr expr is null", K(ret)); - } else if (aggr_exprs.at(i)->is_nested_aggr()) { + } else if (aggr_exprs.at(i)->in_inner_stmt()) { /*do nothing.*/ } else { + // outer aggr int64_t N = aggr_exprs.at(i)->get_param_count(); for (int64_t j = 0; OB_SUCC(ret) && j < N; ++j) { if (OB_ISNULL(aggr_exprs.at(i)->get_param_expr(j))) { @@ -4242,6 +4269,8 @@ int ObTransformPreProcess::generate_child_level_aggr_stmt(ObSelectStmt *select_s { int ret = OB_SUCCESS; ObSEArray complex_aggr_exprs; + ObSEArray inner_aggr_exprs; + if (OB_ISNULL(ctx_) || OB_ISNULL(ctx_->stmt_factory_) || OB_ISNULL(ctx_->expr_factory_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("select stmt is null", K(ret)); @@ -4256,24 +4285,23 @@ int ObTransformPreProcess::generate_child_level_aggr_stmt(ObSelectStmt *select_s ctx_->src_qb_name_, ctx_->src_hash_val_))) { LOG_WARN("failed to recursive adjust statement id", K(ret)); - } else if (OB_FAIL(get_first_level_output_exprs(sub_stmt, - complex_aggr_exprs))) { - LOG_WARN("failed to extract levels aggr.", K(ret)); + } else if (OB_FAIL(get_first_level_output_exprs(sub_stmt, complex_aggr_exprs))) { + LOG_WARN("failed to low levels aggr.", K(ret)); + } else if (OB_FAIL(get_inner_aggr_exprs(sub_stmt, inner_aggr_exprs)) ) { } else { sub_stmt->get_aggr_items().reset(); sub_stmt->get_select_items().reset(); + sub_stmt->get_order_items().reset(); } - for (int64_t i = 0; OB_SUCC(ret) && i < sub_stmt->get_having_exprs().count(); ++i) { - if (OB_FAIL(ObTransformUtils::extract_aggr_expr(sub_stmt->get_having_exprs().at(i), - sub_stmt->get_aggr_items()))) { - LOG_WARN("failed to extract aggr exprs.", K(ret)); - } else { /*do nothing.*/ } + + for (int64_t j = 0; OB_SUCC(ret) && j < inner_aggr_exprs.count(); ++j) { + if (OB_FAIL(add_var_to_array_no_dup(sub_stmt->get_aggr_items(), inner_aggr_exprs.at(j)))) { + LOG_WARN("failed to add aggr exprs to aggr item.", K(ret)); + } } + for (int64_t j = 0; OB_SUCC(ret) && j < complex_aggr_exprs.count(); ++j) { - if (OB_FAIL(ObTransformUtils::extract_aggr_expr(complex_aggr_exprs.at(j), - sub_stmt->get_aggr_items()))) { - LOG_WARN("failed to extract aggr exprs.", K(ret)); - } else if (OB_FAIL(ObTransformUtils::create_select_item(*ctx_->allocator_, + if (OB_FAIL(ObTransformUtils::create_select_item(*ctx_->allocator_, complex_aggr_exprs.at(j), sub_stmt))) { LOG_WARN("failed to push back into select item array.", K(ret)); @@ -4307,7 +4335,7 @@ int ObTransformPreProcess::remove_nested_aggr_exprs(ObSelectStmt *stmt) if (OB_ISNULL(stmt->get_aggr_items().at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("expr of aggr expr is null", K(ret)); - } else if (stmt->get_aggr_items().at(i)->is_nested_aggr()) { + } else if (stmt->get_aggr_items().at(i)->in_inner_stmt()) { /*do nothing.*/ } else if (OB_FAIL(aggr_exprs.push_back(stmt->get_aggr_items().at(i)))) { LOG_WARN("failed to assign to inner aggr exprs.", K(ret)); @@ -4356,7 +4384,6 @@ int ObTransformPreProcess::generate_parent_level_aggr_stmt(ObSelectStmt *&select select_stmt->get_table_items().reset(); select_stmt->get_joined_tables().reset(); select_stmt->get_from_items().reset(); - select_stmt->get_order_items().reset(); select_stmt->get_column_items().reset(); select_stmt->get_condition_exprs().reset(); select_stmt->get_part_exprs().reset(); @@ -4367,6 +4394,8 @@ int ObTransformPreProcess::generate_parent_level_aggr_stmt(ObSelectStmt *&select select_stmt->get_rollup_items().reset(); select_stmt->get_cube_items().reset(); select_stmt->get_having_exprs().reset(); + select_stmt->get_order_items().reset(); + if (OB_FAIL(get_first_level_output_exprs(select_stmt, old_exprs))) { LOG_WARN("failed to get column exprs from stmt from.", K(ret)); diff --git a/src/sql/rewrite/ob_transform_pre_process.h b/src/sql/rewrite/ob_transform_pre_process.h index f4eac63380..4639502884 100644 --- a/src/sql/rewrite/ob_transform_pre_process.h +++ b/src/sql/rewrite/ob_transform_pre_process.h @@ -365,6 +365,7 @@ struct DistinctObjMeta int transform_exprs(ObDMLStmt *stmt, bool &trans_happened); int transform_for_nested_aggregate(ObDMLStmt *&stmt, bool &trans_happened); int generate_child_level_aggr_stmt(ObSelectStmt *stmt, ObSelectStmt *&sub_stmt); + int get_inner_aggr_exprs(ObSelectStmt *sub_stmt, common::ObIArray& inner_aggr_exprs); int get_first_level_output_exprs(ObSelectStmt *sub_stmt, common::ObIArray& inner_aggr_exprs); int generate_parent_level_aggr_stmt(ObSelectStmt *&stmt, ObSelectStmt *sub_stmt); diff --git a/src/sql/rewrite/ob_transform_predicate_move_around.cpp b/src/sql/rewrite/ob_transform_predicate_move_around.cpp index 015ff57a2f..8891875d6f 100644 --- a/src/sql/rewrite/ob_transform_predicate_move_around.cpp +++ b/src/sql/rewrite/ob_transform_predicate_move_around.cpp @@ -118,6 +118,9 @@ int ObTransformPredicateMoveAround::inner_do_transfrom(ObDMLStmt *stmt, bool &tr LOG_WARN("failed to create equal exprs for insert", K(ret)); } else if (OB_FAIL(pushdown_predicates(stmt, dummy_pushdown))) { LOG_WARN("failed to push down predicates", K(ret)); + } else if (real_happened_ && + OB_FAIL(stmt->formalize_stmt_expr_reference(ctx_->expr_factory_, ctx_->session_info_))) { + LOG_WARN("formalize stmt expr reference failed", K(ret)); } else { trans_happened = real_happened_; } diff --git a/src/sql/session/ob_basic_session_info.cpp b/src/sql/session/ob_basic_session_info.cpp index c562657af0..82bf237dea 100644 --- a/src/sql/session/ob_basic_session_info.cpp +++ b/src/sql/session/ob_basic_session_info.cpp @@ -334,6 +334,9 @@ void ObBasicSessionInfo::reset(bool skip_sys_var) MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); if (OB_SUCC(guard.switch_to(tx_desc_->get_tenant_id(), false))) { MTL(transaction::ObTransService*)->release_tx(*tx_desc_); + } else { + LOG_WARN("tenant env not exist, force release tx", KP(tx_desc_), K(tx_desc_->get_tx_id())); + transaction::ObTransService::force_release_tx_when_session_destroy(*tx_desc_); } tx_desc_ = NULL; } diff --git a/src/sql/session/ob_sql_session_info.cpp b/src/sql/session/ob_sql_session_info.cpp index d6569ba93c..b6ab30674b 100644 --- a/src/sql/session/ob_sql_session_info.cpp +++ b/src/sql/session/ob_sql_session_info.cpp @@ -1957,7 +1957,8 @@ const ObAuditRecordData &ObSQLSessionInfo::get_final_audit_record( || EXECUTE_PS_GET_PIECE == mode || EXECUTE_PS_SEND_LONG_DATA == mode || EXECUTE_PS_FETCH == mode - || (EXECUTE_PL_EXECUTE == mode && audit_record_.sql_len_ > 0)) { + || (EXECUTE_PL_EXECUTE == mode && audit_record_.sql_len_ > 0 + && audit_record_.plan_id_ == 0)) { // spi_cursor_open may not use process_record to set audit_record_.sql_ // so only EXECUTE_PL_EXECUTE == mode && audit_record_.sql_len_ > 0 do not set sql //ps模式对应的sql在协议层中设置, session的current_query_中没值 diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 60c4a4f2e3..07298df633 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -324,6 +324,7 @@ ob_set_subtarget(ob_storage tx tx/ob_id_service.cpp tx/ob_keep_alive_ls_handler.cpp tx/ob_tx_loop_worker.cpp + tx/ob_tx_ls_state_mgr.cpp tx/ob_location_adapter.cpp tx/ob_one_phase_committer.cpp tx/ob_standby_timestamp_service.cpp @@ -367,6 +368,7 @@ ob_set_subtarget(ob_storage tx tx/ob_tx_2pc_msg_handler.cpp tx/ob_tx_2pc_ctx_impl.cpp tx/ob_tx_data_define.cpp + tx/ob_tx_data_op.cpp tx/ob_tx_data_functor.cpp tx/ob_tx_serialization.cpp tx/ob_tx_log.cpp diff --git a/src/storage/access/ob_multiple_merge.cpp b/src/storage/access/ob_multiple_merge.cpp index 79b6ab208c..dce8771db4 100644 --- a/src/storage/access/ob_multiple_merge.cpp +++ b/src/storage/access/ob_multiple_merge.cpp @@ -393,7 +393,7 @@ int ObMultipleMerge::project2output_exprs(ObDatumRow &unprojected_row, ObDatumRo nop_pos_.nops_, nop_pos_.count_))) { LOG_WARN("Fail to project row", K(unprojected_row), K(cur_row)); } else { - LOG_DEBUG("Project row", K(unprojected_row), K(cur_row)); + LOG_DEBUG("Project row", K(unprojected_row)); } return ret; } diff --git a/src/storage/access/ob_table_estimator.cpp b/src/storage/access/ob_table_estimator.cpp index 012f228a7c..df13d24a9b 100644 --- a/src/storage/access/ob_table_estimator.cpp +++ b/src/storage/access/ob_table_estimator.cpp @@ -159,14 +159,14 @@ int ObTableEstimator::estimate_multi_scan_row_count( ObPartitionEst sub_range_cost; const static int64_t sub_range_cnt = 3; ObSEArray store_ranges; - if (OB_FAIL((static_cast(current_table))->get_split_ranges( - &range.get_start_key().get_store_rowkey(), - &range.get_end_key().get_store_rowkey(), - sub_range_cnt, - store_ranges))) { - if (OB_ENTRY_NOT_EXIST != ret) { - LOG_WARN("Failed to split ranges", K(ret), K(tmp_cost)); - } + ObStoreRange input_range; + input_range.set_start_key(range.get_start_key().get_store_rowkey()); + input_range.set_end_key(range.get_end_key().get_store_rowkey()); + range.is_left_closed() ? input_range.set_left_closed() : input_range.set_left_open(); + range.is_right_closed() ? input_range.set_right_closed() : input_range.set_right_open(); + if (OB_FAIL((static_cast(current_table)) + ->get_split_ranges(input_range, sub_range_cnt, store_ranges))) { + LOG_WARN("Failed to split ranges", K(ret), K(tmp_cost)); } else if (store_ranges.count() > 1) { LOG_TRACE("estimated logical row count may be not right, split range and do estimating again", K(tmp_cost), K(store_ranges)); common::ObArenaAllocator allocator("OB_STORAGE_EST", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); diff --git a/src/storage/backup/ob_backup_task.cpp b/src/storage/backup/ob_backup_task.cpp index 4e25c91c81..97a7275537 100644 --- a/src/storage/backup/ob_backup_task.cpp +++ b/src/storage/backup/ob_backup_task.cpp @@ -575,7 +575,7 @@ int ObLSBackupDataDagNet::inner_init_before_run_() LOG_WARN("failed to prepare backup tablet provider", K(ret), K(backup_param), K_(backup_data_type)); } else if (OB_FAIL(get_batch_size_(batch_size))) { LOG_WARN("failed to get batch size", K(ret)); - } else if (OB_FAIL(task_mgr_.init(backup_data_type_, batch_size))) { + } else if (OB_FAIL(task_mgr_.init(backup_data_type_, batch_size, ls_backup_ctx_))) { LOG_WARN("failed to init task mgr", K(ret), K(batch_size)); } else { ls_backup_ctx_.stat_mgr_.mark_begin(backup_data_type_); diff --git a/src/storage/backup/ob_backup_utils.cpp b/src/storage/backup/ob_backup_utils.cpp index 512b412ded..027a107efa 100644 --- a/src/storage/backup/ob_backup_utils.cpp +++ b/src/storage/backup/ob_backup_utils.cpp @@ -1761,7 +1761,7 @@ int ObBackupTabletProvider::prepare_tablet_(const uint64_t tenant_id, const shar } else if (!tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("get invalid args", K(ret), K(tablet_id)); - } else if (OB_FAIL(get_tablet_handle_without_memtables_(tenant_id, ls_id, tablet_id, tablet_ref))) { + } else if (OB_FAIL(get_tablet_handle_(tenant_id, ls_id, tablet_id, tablet_ref))) { if (OB_TABLET_NOT_EXIST == ret) { LOG_WARN("failed to get tablet handle", K(ret), K(tenant_id), K(ls_id), K(tablet_id)); ret = OB_SUCCESS; @@ -1785,9 +1785,6 @@ int ObBackupTabletProvider::prepare_tablet_(const uint64_t tenant_id, const shar LOG_WARN("can not backup replica", K(ret), K(tablet_id), K(ls_id)); } else if (OB_FAIL(check_tablet_replica_validity_(tenant_id, ls_id, tablet_id, backup_data_type))) { LOG_WARN("failed to check tablet replica validity", K(ret), K(tenant_id), K(ls_id), K(tablet_id), K(backup_data_type)); - } else if (OB_FAIL(hold_tablet_handle_(tablet_id, tablet_ref))) { - ls_backup_ctx_->tablet_holder_.free_tablet_ref(tablet_ref); - LOG_WARN("failed to hold tablet handle", K(ret), K(tablet_id), KPC(tablet_ref)); } else if (OB_FAIL(tablet_ref->tablet_handle_.get_obj()->fetch_table_store(table_store_wrapper))) { LOG_WARN("fail to fetch table store", K(ret)); } else if (OB_FAIL(fetch_tablet_sstable_array_( @@ -1917,10 +1914,11 @@ int ObBackupTabletProvider::get_consistent_scn_(share::SCN &consistent_scn) cons } int ObBackupTabletProvider::get_tablet_handle_(const uint64_t tenant_id, const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, ObTabletHandle &tablet_handle) + const common::ObTabletID &tablet_id, ObBackupTabletHandleRef *&tablet_ref) { int ret = OB_SUCCESS; - tablet_handle.reset(); + tablet_ref = NULL; + bool hold_tablet_success = false; if (OB_INVALID_ID == tenant_id || !ls_id.is_valid() || !tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("get invalid args", K(ret), K(tenant_id), K(ls_id), K(tablet_id)); @@ -1930,30 +1928,19 @@ int ObBackupTabletProvider::get_tablet_handle_(const uint64_t tenant_id, const s } else { const int64_t rebuild_seq = ls_backup_ctx_->rebuild_seq_; MTL_SWITCH(tenant_id) { - ObLS *ls = NULL; - ObLSHandle ls_handle; - ObLSService *ls_svr = NULL; - if (OB_ISNULL(ls_svr = MTL(ObLSService *))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("MTL ObLSService is null", K(ret), K(tenant_id)); - } else if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("fail to get ls handle", K(ret), K(ls_id)); - } else if (OB_ISNULL(ls = ls_handle.get_ls())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("LS is null", K(ret)); - } else if (OB_FAIL(ObBackupUtils::check_ls_valid_for_backup(tenant_id, ls_id, rebuild_seq))) { + if (OB_FAIL(ObBackupUtils::check_ls_valid_for_backup(tenant_id, ls_id, rebuild_seq))) { LOG_WARN("failed to check ls valid for backup", K(ret), K(tenant_id), K(ls_id), K(rebuild_seq)); } else { // sync wait transfer in tablet replace table finish const int64_t ABS_TIMEOUT_TS = ObTimeUtil::current_time() + 5 * 60 * 1000 * 1000; //5min bool is_normal_tablet = false; while (OB_SUCC(ret)) { - tablet_handle.reset(); + tablet_ref = NULL; if (ABS_TIMEOUT_TS < ObTimeUtil::current_time()) { ret = OB_TIMEOUT; LOG_WARN("backup get tablet handle timeout", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(ls->get_tablet(tablet_id, tablet_handle))) {// read readble commited, only get NORMAL and TRANSFER IN tablet. - LOG_WARN("failed to get tablet handle", K(ret), K(tenant_id), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(inner_get_tablet_handle_without_memtables_(tenant_id, ls_id, tablet_id, tablet_ref))) { // read readble commited, only get NORMAL and TRANSFER IN tablet. + LOG_WARN("failed to inner get tablet handle without memtables", K(ret), K(tenant_id), K(ls_id), K(tablet_id)); } else if (OB_FAIL(ObBackupUtils::check_ls_valid_for_backup(tenant_id, ls_id, rebuild_seq))) { LOG_WARN("failed to check ls valid for backup", K(ret), K(tenant_id), K(ls_id), K(rebuild_seq)); } else if (tablet_id.is_ls_inner_tablet()) { @@ -1961,10 +1948,10 @@ int ObBackupTabletProvider::get_tablet_handle_(const uint64_t tenant_id, const s // Data of inner tablets is backed up with meta at the same replica. And. // the clog_checkpoint_scn < consistent_scn is allowed. break; - } else if (tablet_handle.get_obj()->get_tablet_meta().has_transfer_table()) { + } else if (tablet_ref->tablet_handle_.get_obj()->get_tablet_meta().has_transfer_table()) { LOG_INFO("transfer table is not replaced", K(ret), K(tenant_id), K(ls_id), K(tablet_id)); usleep(100 * 1000); // wait 100ms - } else if (OB_FAIL(check_tablet_status_(tablet_handle, is_normal_tablet))) { + } else if (OB_FAIL(check_tablet_status_(tablet_ref->tablet_handle_, is_normal_tablet))) { LOG_WARN("failed to check tablet is normal", K(ret), K(tenant_id), K(ls_id), K(rebuild_seq)); } else if (!is_normal_tablet) { LOG_INFO("tablet status is not normal", K(tenant_id), K(ls_id), K(tablet_id)); @@ -1972,14 +1959,25 @@ int ObBackupTabletProvider::get_tablet_handle_(const uint64_t tenant_id, const s } else { break; } + if (OB_NOT_NULL(tablet_ref)) { + ls_backup_ctx_->tablet_holder_.free_tablet_ref(tablet_ref); + } } } } + if (FAILEDx(hold_tablet_handle_(tablet_id, tablet_ref))) { + LOG_WARN("failed to hold tablet handle", K(ret), K(tablet_id), KPC(tablet_ref)); + } else { + hold_tablet_success = true; + } + } + if (OB_NOT_NULL(ls_backup_ctx_) && OB_NOT_NULL(tablet_ref) && !hold_tablet_success) { + ls_backup_ctx_->tablet_holder_.free_tablet_ref(tablet_ref); } return ret; } -int ObBackupTabletProvider::get_tablet_handle_without_memtables_(const uint64_t tenant_id, const share::ObLSID &ls_id, +int ObBackupTabletProvider::inner_get_tablet_handle_without_memtables_(const uint64_t tenant_id, const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, ObBackupTabletHandleRef *&tablet_ref) { int ret = OB_SUCCESS; @@ -2524,7 +2522,8 @@ ObBackupMacroBlockTaskMgr::ObBackupMacroBlockTaskMgr() file_id_(0), cur_task_id_(0), pending_list_(), - ready_list_() + ready_list_(), + ls_backup_ctx_(NULL) {} ObBackupMacroBlockTaskMgr::~ObBackupMacroBlockTaskMgr() @@ -2532,7 +2531,8 @@ ObBackupMacroBlockTaskMgr::~ObBackupMacroBlockTaskMgr() reset(); } -int ObBackupMacroBlockTaskMgr::init(const share::ObBackupDataType &backup_data_type, const int64_t batch_size) +int ObBackupMacroBlockTaskMgr::init(const share::ObBackupDataType &backup_data_type, const int64_t batch_size, + ObLSBackupCtx &ls_backup_ctx) { int ret = OB_SUCCESS; if (IS_INIT) { @@ -2553,6 +2553,7 @@ int ObBackupMacroBlockTaskMgr::init(const share::ObBackupDataType &backup_data_t #endif max_task_id_ = 0; cur_task_id_ = 0; + ls_backup_ctx_ = &ls_backup_ctx; is_inited_ = true; } return ret; @@ -2596,7 +2597,14 @@ int ObBackupMacroBlockTaskMgr::deliver(common::ObIArray &i ObThreadCondGuard guard(cond_); int64_t begin_ms = ObTimeUtility::fast_current_time(); while (OB_SUCC(ret) && id_list.empty()) { - if (OB_FAIL(get_from_ready_list_(id_list))) { + if (OB_ISNULL(ls_backup_ctx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls backup ctx should not be null", K(ret)); + } else if (OB_SUCCESS != ls_backup_ctx_->get_result_code()) { + ret = ls_backup_ctx_->get_result_code(); + LOG_INFO("ls backup ctx already failed", K(ret)); + break; + } else if (OB_FAIL(get_from_ready_list_(id_list))) { LOG_WARN("failed to get from ready list", K(ret)); } else if (!id_list.empty()) { break; diff --git a/src/storage/backup/ob_backup_utils.h b/src/storage/backup/ob_backup_utils.h index 7a2a102ce9..58122ed252 100644 --- a/src/storage/backup/ob_backup_utils.h +++ b/src/storage/backup/ob_backup_utils.h @@ -299,8 +299,8 @@ private: // make sure clog checkpoint scn of the returned tablet is >= consistent_scn. int get_tablet_handle_(const uint64_t tenant_id, const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, - storage::ObTabletHandle &tablet_handle); - int get_tablet_handle_without_memtables_(const uint64_t tenant_id, const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, + ObBackupTabletHandleRef *&tablet_ref); + int inner_get_tablet_handle_without_memtables_(const uint64_t tenant_id, const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, ObBackupTabletHandleRef *&tablet_ref); int get_consistent_scn_(share::SCN &consistent_scn) const; int report_tablet_skipped_(const common::ObTabletID &tablet_id, const share::ObBackupSkippedType &skipped_type, @@ -368,7 +368,8 @@ class ObBackupMacroBlockTaskMgr { public: ObBackupMacroBlockTaskMgr(); virtual ~ObBackupMacroBlockTaskMgr(); - int init(const share::ObBackupDataType &backup_data_type, const int64_t batch_size); + int init(const share::ObBackupDataType &backup_data_type, const int64_t batch_size, + ObLSBackupCtx &ls_backup_ctx); void set_backup_data_type(const share::ObBackupDataType &backup_data_type); share::ObBackupDataType get_backup_data_type() const; int receive(const int64_t task_id, const common::ObIArray &id_list); @@ -402,6 +403,7 @@ private: volatile int64_t cur_task_id_; ObArray pending_list_; ObArray ready_list_; + ObLSBackupCtx *ls_backup_ctx_; DISALLOW_COPY_AND_ASSIGN(ObBackupMacroBlockTaskMgr); }; diff --git a/src/storage/blocksstable/ob_block_manager.cpp b/src/storage/blocksstable/ob_block_manager.cpp index 771cfe5d87..3101a4a6d9 100644 --- a/src/storage/blocksstable/ob_block_manager.cpp +++ b/src/storage/blocksstable/ob_block_manager.cpp @@ -1006,8 +1006,14 @@ void ObBlockManager::mark_and_sweep() } else if (0 == mark_info.count()) { skip_mark = true; LOG_INFO("no block alloc/free, no need to mark blocks", K(ret), K(mark_info.count())); - } else if (OB_FAIL(mark_macro_blocks(mark_info, macro_id_set, tmp_status))) {//mark - LOG_WARN("fail to mark macro blocks", K(ret)); + } else if (OB_FAIL(mark_macro_blocks(mark_info, macro_id_set, tmp_status))) { + if (OB_ALLOCATE_MEMORY_FAILED == ret) { + LOG_INFO("mark blocks meet memory issue, still countinue sweep to lease compaction space"); + ret = OB_SUCCESS; + skip_mark = true; + } else { + LOG_WARN("fail to mark macro blocks", K(ret)); + } } if (OB_FAIL(ret)) { diff --git a/src/storage/blocksstable/ob_sstable_meta.cpp b/src/storage/blocksstable/ob_sstable_meta.cpp index 9582273c40..f5d197fa6c 100644 --- a/src/storage/blocksstable/ob_sstable_meta.cpp +++ b/src/storage/blocksstable/ob_sstable_meta.cpp @@ -374,6 +374,182 @@ int ObSSTableBasicMeta::set_upper_trans_version(const int64_t upper_trans_versio return ret; } +//================================== ObTxDesc & ObTxContext ================================== +int ObTxContext::ObTxDesc::serialize(char *buf, const int64_t buf_len, int64_t &pos) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(serialization::encode_i64(buf, buf_len,pos, tx_id_))) { + LOG_WARN("fail to encode length", K(ret), K(buf_len), K(pos)); + } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, pos, row_count_))) { + LOG_WARN("fail to encode length", K(ret), K(buf_len), K(pos)); + } + return ret; +} + +int ObTxContext::ObTxDesc::deserialize(const char *buf, const int64_t buf_len, int64_t &pos) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(serialization::decode_i64(buf, buf_len, pos, &tx_id_))) { + LOG_WARN("fail to encode length", K(ret), K(buf_len), K(pos)); + } else if (OB_FAIL(serialization::decode_i64(buf, buf_len, pos, &row_count_))) { + LOG_WARN("fail to encode length", K(ret), K(buf_len), K(pos)); + } + return ret; +} + +int64_t ObTxContext::ObTxDesc::get_serialize_size() const +{ + return serialization::encoded_length_i64(tx_id_) + serialization::encoded_length_i64(row_count_); +} + +int ObTxContext::serialize(char *buf, const int64_t buf_len, int64_t &pos) const +{ + int ret = OB_SUCCESS; + const int64_t tmp_pos = pos; + const_cast(this)->len_ = get_serialize_size(); + if (OB_FAIL(serialization::encode_i32(buf, buf_len,pos, len_))) { + LOG_WARN("fail to encode length", K(ret), K(buf_len), K(pos)); + } else if (OB_FAIL(serialization::encode_vi64(buf, buf_len, pos, count_))) { + LOG_WARN("fail to encode count", K(ret), K(buf_len), K(pos)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < count_; i ++) { + if (OB_FAIL(serialization::encode(buf, buf_len, pos, tx_descs_[i]))) { + LOG_WARN("fail to encode item", K(i), K(ret), K(buf_len), K(pos)); + } + } + + if (OB_FAIL(ret)) { + } else if (OB_UNLIKELY(pos - tmp_pos != len_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected len_", K(ret), K(len_), K(tmp_pos), K(pos)); + } + return ret; +} + +int64_t ObTxContext::get_serialize_size() const +{ + int64_t size = serialization::encoded_length_i32(len_); + size += serialization::encoded_length_vi64(count_); + for (int64_t i = 0; i < count_; i++) { + size += serialization::encoded_length(tx_descs_[i]); + } + return size; +} + +int ObTxContext::deserialize( + common::ObArenaAllocator &allocator, + const char *buf, + const int64_t buf_len, + int64_t &pos) +{ + int ret = OB_SUCCESS; + const int64_t tmp_pos = pos; + if (OB_FAIL(serialization::decode_i32(buf, buf_len, pos, &len_))) { + LOG_WARN("fail to encode length", K(ret), K(buf_len), K(pos)); + } else if (OB_FAIL(serialization::decode_vi64(buf, buf_len, pos, &count_))) { + LOG_WARN("fail to decode ob array count", K(ret)); + } else if (count_ > 0) { + if (OB_ISNULL(tx_descs_ = static_cast(allocator.alloc(sizeof(ObTxDesc) * count_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate tx context", K(ret), K(count_)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < count_; i ++) { + ObTxDesc &item = tx_descs_[i]; + if (OB_FAIL(serialization::decode(buf, buf_len, pos, item))) { + LOG_WARN("fail to decode array item", K(ret), K(i), K(count_)); + } + } + + if (OB_FAIL(ret)) { + } else if (OB_UNLIKELY(pos - tmp_pos != len_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected len_", K(ret), K(len_), K(tmp_pos), K(pos)); + } + return ret; +} + +int ObTxContext::deep_copy( + char *buf, + const int64_t buf_len, + int64_t &pos, + ObTxContext &dest) const +{ + int ret = OB_SUCCESS; + const int64_t variable_size = get_variable_size(); + if (this == &dest) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("can't deep copy self", K(ret), K(*this)); + } else if (pos + variable_size > buf_len) { + ret = OB_BUF_NOT_ENOUGH; + LOG_WARN("buf not enough", K(ret), K(pos), K(buf_len), K(*this)); + } else { + dest.len_ = len_; + dest.count_ = count_; + if (0 == count_) { + dest.tx_descs_ = nullptr; + } else { + dest.tx_descs_ = reinterpret_cast(buf + pos); + for (int64_t i = 0; i < count_; i++) { + dest.tx_descs_[i] = tx_descs_[i]; + } + } + pos += variable_size; + } + return ret; +} + +int ObTxContext::init(const common::ObIArray &tx_descs, common::ObArenaAllocator &allocator) +{ + int ret = OB_SUCCESS; + const int64_t cnt = tx_descs.count(); + if (nullptr != tx_descs_ || count_ > 0) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", K(ret), K(*this)); + } else if (OB_UNLIKELY(MAX_TX_IDS_COUNT < cnt)) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("too many tx desc", K(ret), K(cnt)); + } else if (0 == cnt) { + reset(); + } else if (OB_ISNULL(tx_descs_ = static_cast(allocator.alloc(sizeof(ObTxContext::ObTxDesc) * cnt)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate tx context", K(ret), KP(tx_descs_)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < cnt; i++) { + if (OB_FAIL(push_back(tx_descs.at(i)))) { + LOG_WARN("failed to alloc memory for tx_ids_", K(ret), K(i), K(tx_descs.at(i))); + } + } + } + return ret; +} + +int64_t ObTxContext::get_tx_id(const int64_t idx) const +{ + OB_ASSERT(idx >=0 && idx < count_); + return tx_descs_[idx].tx_id_; +} + +int ObTxContext::push_back(const ObTxDesc &desc) +{ + int ret = OB_SUCCESS; + if (count_ >= MAX_TX_IDS_COUNT) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("tx desc array overflow", K(ret), K(count_)); + } else if (OB_ISNULL(tx_descs_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tx desc array is null", K(ret), K(count_), KP(tx_descs_)); + } else { + tx_descs_[count_++] = desc; + } + return ret; +} + +int64_t ObTxContext::get_variable_size() const +{ + return count_ * sizeof(ObTxDesc); +} + //================================== ObSSTableMeta ================================== ObSSTableMeta::ObSSTableMeta() : basic_meta_(), @@ -507,6 +683,20 @@ int ObSSTableMeta::prepare_column_checksum( return ret; } +int ObSSTableMeta::prepare_tx_context( + const ObTxContext::ObTxDesc &tx_desc, + common::ObArenaAllocator &allocator) +{ + int ret = OB_SUCCESS; + ObSEArray tx_desc_arr; + if (OB_FAIL(tx_desc_arr.push_back(tx_desc))) { + LOG_WARN("push back tx desc fail", K(ret), K(tx_desc)); + } else if (OB_FAIL(tx_ctx_.init(tx_desc_arr, allocator))) { + LOG_WARN("failed to alloc memory for tx_ids_", K(ret), K(tx_desc)); + } + return ret; +} + bool ObSSTableMeta::check_meta() const { return basic_meta_.is_valid() @@ -548,7 +738,7 @@ int ObSSTableMeta::init( } if (OB_SUCC(ret) && transaction::ObTransID(param.uncommitted_tx_id_).is_valid()) { - if (OB_FAIL(tx_ctx_.tx_descs_.push_back({param.uncommitted_tx_id_, 0}))) { + if (OB_FAIL(prepare_tx_context({param.uncommitted_tx_id_, 0}, allocator))) { LOG_WARN("failed to alloc memory for tx_ids_", K(ret), K(param)); } } @@ -694,7 +884,7 @@ int ObSSTableMeta::deserialize_( LOG_WARN("fail to deserialize macro info", K(ret), K(data_len), K(pos), K(des_meta)); } else if (pos < data_len && OB_FAIL(cg_sstables_.deserialize(allocator, buf, data_len, pos))) { LOG_WARN("fail to deserialize cg sstables", K(ret), K(data_len), K(pos)); - } else if (pos < data_len && OB_FAIL(tx_ctx_.deserialize(buf, data_len, pos))) { + } else if (pos < data_len && OB_FAIL(tx_ctx_.deserialize(allocator, buf, data_len, pos))) { LOG_WARN("fail to deserialize tx ids", K(ret), K(data_len), K(pos)); } } @@ -729,7 +919,8 @@ int64_t ObSSTableMeta::get_variable_size() const return sizeof(int64_t) * column_checksum_count_ // column checksums + data_root_info_.get_variable_size() + macro_info_.get_variable_size() - + cg_sstables_.get_deep_copy_size(); + + cg_sstables_.get_deep_copy_size() + + tx_ctx_.get_variable_size(); } int ObSSTableMeta::deep_copy( @@ -759,8 +950,12 @@ int ObSSTableMeta::deep_copy( LOG_WARN("fail to deep copy macro info", K(ret), KP(buf), K(buf_len), K(pos), K(macro_info_)); } else if (OB_FAIL(cg_sstables_.deep_copy(buf, buf_len, pos, dest->cg_sstables_))) { LOG_WARN("fail to deep copy cg sstables", K(ret), KP(buf), K(buf_len), K(pos), K(cg_sstables_)); - } else if (OB_FAIL(dest->tx_ctx_.assign(tx_ctx_))) { - LOG_WARN("fail to deep copy cg sstables", K(ret), K(tx_ctx_)); + } else if (OB_FAIL(tx_ctx_.deep_copy(buf, buf_len, pos, dest->tx_ctx_))) { + LOG_WARN("fail to deep copy tx context", K(ret), K(tx_ctx_)); + // TODO (jiahua.cjh): add defend code back + // } else if (deep_size != pos - tmp_pos) { + // ret = OB_ERR_UNEXPECTED; + // LOG_WARN("deep copy size miss match", K(ret), K(*this), KPC(dest), K(deep_size), K(tmp_pos), K(pos)); } else { dest->is_inited_ = is_inited_; } diff --git a/src/storage/blocksstable/ob_sstable_meta.h b/src/storage/blocksstable/ob_sstable_meta.h index ac833c8539..f6bbe91368 100644 --- a/src/storage/blocksstable/ob_sstable_meta.h +++ b/src/storage/blocksstable/ob_sstable_meta.h @@ -13,6 +13,7 @@ #ifndef OCEANBASE_STORAGE_BLOCKSSTABLE_OB_SSTABLE_META_H #define OCEANBASE_STORAGE_BLOCKSSTABLE_OB_SSTABLE_META_H +#include "lib/container/ob_iarray.h" #include "share/schema/ob_table_schema.h" #include "storage/ob_storage_schema.h" #include "storage/ob_i_table.h" @@ -29,80 +30,49 @@ struct ObTabletCreateSSTableParam; } namespace blocksstable { -struct ObTxContext +class ObTxContext final { - struct ObTxDesc{ +public: + struct ObTxDesc final + { int64_t tx_id_; int64_t row_count_; - int serialize(char *buf, const int64_t buf_len, int64_t &pos) const - { - int ret = OB_SUCCESS; - if (OB_FAIL(serialization::encode_i64(buf, buf_len,pos, tx_id_))) { - STORAGE_LOG(WARN, "fail to encode length", K(ret), K(buf_len), K(pos)); - } else if (OB_FAIL(serialization::encode_i64(buf, buf_len, pos, row_count_))) { - STORAGE_LOG(WARN, "fail to encode length", K(ret), K(buf_len), K(pos)); - } - return ret; - } - - int deserialize(const char *buf, const int64_t buf_len, int64_t &pos) - { - int ret = OB_SUCCESS; - if (OB_FAIL(serialization::decode_i64(buf, buf_len, pos, &tx_id_))) { - STORAGE_LOG(WARN, "fail to encode length", K(ret), K(buf_len), K(pos)); - } else if (OB_FAIL(serialization::decode_i64(buf, buf_len, pos, &row_count_))) { - STORAGE_LOG(WARN, "fail to encode length", K(ret), K(buf_len), K(pos)); - } - return ret; - } - - int64_t get_serialize_size() const { - return serialization::encoded_length_i64(tx_id_) + serialization::encoded_length_i64(row_count_); - } + int serialize(char *buf, const int64_t buf_len, int64_t &pos) const; + int deserialize(const char *buf, const int64_t buf_len, int64_t &pos); + int64_t get_serialize_size() const; TO_STRING_KV(K(tx_id_), K(row_count_)); }; + ObTxContext() : len_(0), count_(0), tx_descs_(nullptr) {}; - int serialize(char *buf, const int64_t buf_len, int64_t &pos) const + int init(const common::ObIArray &tx_descs, common::ObArenaAllocator &allocator); + int serialize(char *buf, const int64_t buf_len, int64_t &pos) const; + int64_t get_serialize_size() const; + int deserialize(common::ObArenaAllocator &allocator, const char *buf, const int64_t buf_len, int64_t &pos); + int deep_copy( + char *buf, + const int64_t buf_len, + int64_t &pos, + ObTxContext &dest) const; + int64_t get_variable_size() const; + OB_INLINE int64_t get_count() const { return count_; } + int64_t get_tx_id(const int64_t idx) const; + void reset() { - int ret = OB_SUCCESS; - const_cast(this)->len_ = get_serialize_size(); - if (OB_FAIL(serialization::encode_i32(buf, buf_len,pos, len_))) { - STORAGE_LOG(WARN, "fail to encode length", K(ret), K(buf_len), K(pos)); - } else if (OB_FAIL(tx_descs_.serialize(buf, buf_len, pos))) { - STORAGE_LOG(WARN, "fail to encode length", K(ret), K(buf_len), K(pos)); - } - return ret; - } - int64_t get_serialize_size() const { - return serialization::encoded_length_i32(len_) + tx_descs_.get_serialize_size(); - } - int deserialize(const char *buf, const int64_t buf_len, int64_t &pos) - { - int ret = OB_SUCCESS; - const int64_t tmp_pos = pos; - if (OB_FAIL(serialization::decode_i32(buf, buf_len, pos, &len_))) { - STORAGE_LOG(WARN, "fail to encode length", K(ret), K(buf_len), K(pos)); - } else if (OB_FAIL(tx_descs_.deserialize(buf, buf_len, pos))) { - STORAGE_LOG(WARN, "fail to encode length", K(ret), K(buf_len), K(pos)); - } else if (OB_UNLIKELY(pos - tmp_pos != len_)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "unexpected len_", K(ret), K(len_), K(tmp_pos), K(pos)); - } - return ret; - } - int assign(const ObTxContext &tx_ctx) { - len_ = tx_ctx.len_; - return tx_descs_.assign(tx_ctx.tx_descs_); - } - void reset() { len_ = 0; - tx_descs_.reset(); + count_ = 0; + tx_descs_ = nullptr; } + TO_STRING_KV(K_(count), K(ObArrayWrap(tx_descs_, count_))); +private: + int push_back(const ObTxDesc &desc); + +private: static const int64_t MAX_TX_IDS_COUNT = 16; - int32_t len_; - ObSEArray tx_descs_; - TO_STRING_KV(K(tx_descs_)); + int32_t len_; // for compat + int64_t count_; // actual item count + ObTxDesc *tx_descs_; + DISALLOW_COPY_AND_ASSIGN(ObTxContext); }; //For compatibility, the variables in this struct MUST NOT be deleted or moved. @@ -230,8 +200,8 @@ public: OB_INLINE const ObSSTableBasicMeta &get_basic_meta() const { return basic_meta_; } OB_INLINE int64_t get_col_checksum_cnt() const { return column_checksum_count_; } OB_INLINE int64_t *get_col_checksum() const { return column_checksums_; } - OB_INLINE int64_t get_tx_id_count() const { return tx_ctx_.tx_descs_.count(); } - OB_INLINE int64_t get_tx_ids(int64_t idx) const { return tx_ctx_.tx_descs_.at(idx).tx_id_; } + OB_INLINE int64_t get_tx_id_count() const { return tx_ctx_.get_count(); } + OB_INLINE int64_t get_tx_ids(const int64_t idx) const { return tx_ctx_.get_tx_id(idx); } OB_INLINE int64_t get_data_checksum() const { return basic_meta_.data_checksum_; } OB_INLINE int64_t get_rowkey_column_count() const { return basic_meta_.rowkey_column_count_; } OB_INLINE int64_t get_column_count() const { return basic_meta_.column_cnt_; } @@ -331,6 +301,9 @@ private: int prepare_column_checksum( const common::ObIArray &column_checksums, common::ObArenaAllocator &allocator); + int prepare_tx_context( + const ObTxContext::ObTxDesc &tx_desc, + common::ObArenaAllocator &allocator); int serialize_(char *buf, const int64_t buf_len, int64_t &pos) const; int deserialize_( common::ObArenaAllocator &allocator, diff --git a/src/storage/compaction/ob_compaction_schedule_util.cpp b/src/storage/compaction/ob_compaction_schedule_util.cpp index 0999393143..b615f1cca5 100644 --- a/src/storage/compaction/ob_compaction_schedule_util.cpp +++ b/src/storage/compaction/ob_compaction_schedule_util.cpp @@ -18,45 +18,5 @@ namespace oceanbase namespace compaction { -/** - * ObCompactionScheduleTimeGuard Impl - */ -const char *ObCompactionScheduleTimeGuard::CompactionEventStr[] = { - "GET_TABLET", - "UPDATE_TABLET_REPORT_STATUS", - "READ_MEDIUM_INFO", - "SCHEDULE_NEXT_MEDIUM", - "SCHEDULE_TABLET_MEDIUM", - "FAST_FREEZE", - "SEARCH_META_TABLE", - "CHECK_META_TABLE", - "SEARCH_CHECKSUM", - "CHECK_CHECKSUM", - "SCHEDULER_NEXT_ROUND" -}; - -const char *ObCompactionScheduleTimeGuard::get_comp_event_str(enum CompactionEvent event) -{ - STATIC_ASSERT(static_cast(COMPACTION_EVENT_MAX) == ARRAYSIZEOF(CompactionEventStr), "events str len is mismatch"); - const char *str = ""; - if (event >= COMPACTION_EVENT_MAX || event < GET_TABLET) { - str = "invalid_type"; - } else { - str = CompactionEventStr[event]; - } - return str; -} - -int64_t ObCompactionScheduleTimeGuard::to_string(char *buf, const int64_t buf_len) const -{ - int64_t pos = 0; - for (int64_t idx = 0; idx < idx_; ++idx) { - if (0 < click_poinsts_[idx]) { - fmt_ts_to_meaningful_str(buf, buf_len, pos, get_comp_event_str((CompactionEvent)line_array_[idx]), click_poinsts_[idx]); - } - } - return pos; -} - } // namespace compaction } // namespace oceanbase \ No newline at end of file diff --git a/src/storage/compaction/ob_compaction_schedule_util.h b/src/storage/compaction/ob_compaction_schedule_util.h index 1f6623d11a..2201aecefc 100644 --- a/src/storage/compaction/ob_compaction_schedule_util.h +++ b/src/storage/compaction/ob_compaction_schedule_util.h @@ -58,35 +58,6 @@ public: int64_t wait_rs_validate_cnt_; }; -struct ObCompactionScheduleTimeGuard : public ObCompactionTimeGuard -{ -public: - ObCompactionScheduleTimeGuard() - : ObCompactionTimeGuard(UINT64_MAX, "[STORAGE] ") - {} - virtual ~ObCompactionScheduleTimeGuard() {} - enum CompactionEvent : uint16_t { - // medium scheduler - GET_TABLET, - UPDATE_TABLET_REPORT_STATUS, - READ_MEDIUM_INFO, - SCHEDULE_NEXT_MEDIUM, - SCHEDULE_TABLET_MEDIUM, - FAST_FREEZE, - // medium checker - SEARCH_META_TABLE, - CHECK_META_TABLE, - SEARCH_CHECKSUM, - CHECK_CHECKSUM, - SCHEDULER_NEXT_ROUND, - COMPACTION_EVENT_MAX - }; - virtual int64_t to_string(char *buf, const int64_t buf_len) const override; -private: - const static char *CompactionEventStr[]; - static const char *get_comp_event_str(enum CompactionEvent event); -}; - } // compaction } // oceanbase diff --git a/src/storage/compaction/ob_partition_parallel_merge_ctx.cpp b/src/storage/compaction/ob_partition_parallel_merge_ctx.cpp index c8dc8b9c49..be6fcd6017 100644 --- a/src/storage/compaction/ob_partition_parallel_merge_ctx.cpp +++ b/src/storage/compaction/ob_partition_parallel_merge_ctx.cpp @@ -287,22 +287,23 @@ int ObParallelMergeCtx::init_parallel_mini_merge(compaction::ObBasicTabletMergeC ObArray store_ranges; store_ranges.set_attr(lib::ObMemAttr(MTL_ID(), "TmpMiniRanges", ObCtxIds::MERGE_NORMAL_CTX_ID)); + ObStoreRange input_range; + input_range.set_whole_range(); if (concurrent_cnt_ <= 1) { if (OB_FAIL(init_serial_merge())) { STORAGE_LOG(WARN, "Failed to init serialize merge", K(ret)); } - } else if (OB_FAIL(memtable->get_split_ranges(nullptr, nullptr, concurrent_cnt_, store_ranges))) { - if (OB_ENTRY_NOT_EXIST == ret) { + } else if (OB_FAIL(memtable->get_split_ranges(input_range, concurrent_cnt_, store_ranges))) { + STORAGE_LOG(WARN, "Failed to get split ranges from memtable", K(ret)); + } else if (OB_UNLIKELY(store_ranges.count() != concurrent_cnt_)) { + if (1 == store_ranges.count()) { if (OB_FAIL(init_serial_merge())) { STORAGE_LOG(WARN, "Failed to init serialize merge", K(ret)); } } else { - STORAGE_LOG(WARN, "Failed to get split ranges from memtable", K(ret)); + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Unexpected range array and concurrent_cnt", K(ret), K_(concurrent_cnt), K(store_ranges)); } - } else if (OB_UNLIKELY(store_ranges.count() != concurrent_cnt_)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "Unexpected range array and concurrent_cnt", K(ret), K_(concurrent_cnt), - K(store_ranges)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < store_ranges.count(); i++) { ObDatumRange datum_range; diff --git a/src/storage/compaction/ob_tablet_merge_task.cpp b/src/storage/compaction/ob_tablet_merge_task.cpp index 9ac144514c..6a3c0e5dbd 100644 --- a/src/storage/compaction/ob_tablet_merge_task.cpp +++ b/src/storage/compaction/ob_tablet_merge_task.cpp @@ -52,71 +52,6 @@ using namespace blocksstable; namespace compaction { -/* - * ----------------------------------------------ObCompactionTimeGuard-------------------------------------------------- - */ -constexpr float ObStorageCompactionTimeGuard::COMPACTION_SHOW_PERCENT_THRESHOLD; -const char *ObStorageCompactionTimeGuard::CompactionEventStr[] = { - "WAIT_TO_SCHEDULE", - "COMPACTION_POLICY", - "PRE_PROCESS_TX_TABLE", - "GET_PARALLEL_RANGE", - "EXECUTE", - "CREATE_SSTABLE", - "UPDATE_TABLET", - "RELEASE_MEMTABLE", - "SCHEDULE_OTHER_COMPACTION", - "DAG_FINISH" -}; - -const char *ObStorageCompactionTimeGuard::get_comp_event_str(enum CompactionEvent event) -{ - STATIC_ASSERT(static_cast(COMPACTION_EVENT_MAX) == ARRAYSIZEOF(CompactionEventStr), "events str len is mismatch"); - const char *str = ""; - if (event >= COMPACTION_EVENT_MAX || event < DAG_WAIT_TO_SCHEDULE) { - str = "invalid_type"; - } else { - str = CompactionEventStr[event]; - } - return str; -} - -int64_t ObStorageCompactionTimeGuard::to_string(char *buf, const int64_t buf_len) const -{ - int64_t pos = 0; - int64_t total_cost = 0; - J_KV(K_(add_time)); - common::databuff_printf(buf, buf_len, pos, "|"); - if (idx_ > DAG_WAIT_TO_SCHEDULE && click_poinsts_[DAG_WAIT_TO_SCHEDULE] > COMPACTION_SHOW_TIME_THRESHOLD) { - fmt_ts_to_meaningful_str(buf, buf_len, pos, "wait_schedule_time", click_poinsts_[DAG_WAIT_TO_SCHEDULE]); - } - for (int64_t idx = COMPACTION_POLICY; idx < idx_; ++idx) { - total_cost += click_poinsts_[idx]; - } - if (total_cost > COMPACTION_SHOW_TIME_THRESHOLD) { - float ratio = 0; - for (int64_t idx = COMPACTION_POLICY; idx < idx_; ++idx) { - const uint32_t time_interval = click_poinsts_[idx]; - ratio = (float)(time_interval)/ total_cost; - if (ratio >= COMPACTION_SHOW_PERCENT_THRESHOLD || time_interval >= COMPACTION_SHOW_TIME_THRESHOLD) { - fmt_ts_to_meaningful_str(buf, buf_len, pos, get_comp_event_str((CompactionEvent)line_array_[idx]), click_poinsts_[idx]); - if (ratio > 0.01) { - common::databuff_printf(buf, buf_len, pos, "(%.2f)", ratio); - } - common::databuff_printf(buf, buf_len, pos, "|"); - } - } - } - fmt_ts_to_meaningful_str(buf, buf_len, pos, "total", total_cost); - if (pos != 0 && pos < buf_len) { - buf[pos - 1] = ';'; - } - - if (pos != 0 && pos < buf_len) { - pos -= 1; - } - return pos; -} /* * ----------------------------------------------ObMergeParameter-------------------------------------------------- */ diff --git a/src/storage/compaction/ob_tablet_merge_task.h b/src/storage/compaction/ob_tablet_merge_task.h index cc6db28fb9..377f625e63 100644 --- a/src/storage/compaction/ob_tablet_merge_task.h +++ b/src/storage/compaction/ob_tablet_merge_task.h @@ -57,35 +57,6 @@ Medium Compaction ObTabletMajorMergeDag ObTabletMajorMergeCtx Tx Table Compaction ObTxTableMergeDag ObTabletMiniMergeCtx/ObTabletExeMergeCtx */ -struct ObStorageCompactionTimeGuard : public ObCompactionTimeGuard -{ -public: - ObStorageCompactionTimeGuard() - : ObCompactionTimeGuard(COMPACTION_WARN_THRESHOLD_RATIO, "[STORAGE] ") - {} - virtual ~ObStorageCompactionTimeGuard() {} - enum CompactionEvent : uint16_t { - DAG_WAIT_TO_SCHEDULE = 0, - COMPACTION_POLICY, - PRE_PROCESS_TX_TABLE, - GET_PARALLEL_RANGE, - EXECUTE, - CREATE_SSTABLE, - UPDATE_TABLET, - RELEASE_MEMTABLE, - SCHEDULE_OTHER_COMPACTION, - DAG_FINISH, - COMPACTION_EVENT_MAX - }; - virtual int64_t to_string(char *buf, const int64_t buf_len) const override; -private: - const static char *CompactionEventStr[]; - static const char *get_comp_event_str(enum CompactionEvent event); - static const int64_t COMPACTION_WARN_THRESHOLD_RATIO = 60 * 1000L * 1000L; // 1 min - static constexpr float COMPACTION_SHOW_PERCENT_THRESHOLD = 0.1; - static const int64_t COMPACTION_SHOW_TIME_THRESHOLD = 1 * 1000L * 1000L; // 1s -}; - struct ObMergeParameter { ObMergeParameter( const ObStaticMergeParam &static_param); diff --git a/src/storage/ddl/ob_ddl_inc_redo_log_writer.cpp b/src/storage/ddl/ob_ddl_inc_redo_log_writer.cpp index 3ea8c401d7..7e83a38fdb 100644 --- a/src/storage/ddl/ob_ddl_inc_redo_log_writer.cpp +++ b/src/storage/ddl/ob_ddl_inc_redo_log_writer.cpp @@ -85,6 +85,7 @@ ObDDLIncRedoLogWriter::ObDDLIncRedoLogWriter() ObDDLIncRedoLogWriter::~ObDDLIncRedoLogWriter() { + ddl_inc_log_handle_.reset(); if (nullptr != buffer_) { ob_free(buffer_); buffer_ = nullptr; @@ -234,10 +235,9 @@ int ObDDLIncRedoLogWriter::wait_inc_redo_log_finish() LOG_WARN("fail to wait io finish", K(ret)); } else if (OB_FAIL(ddl_inc_log_handle_.cb_->get_ret_code())) { LOG_WARN("ddl redo callback executed failed", K(ret)); - } else { - ddl_inc_log_handle_.reset(); } } + ddl_inc_log_handle_.reset(); return ret; } diff --git a/src/storage/ddl/ob_ddl_merge_task.cpp b/src/storage/ddl/ob_ddl_merge_task.cpp index 7deeee68bd..b9bd21e2ce 100644 --- a/src/storage/ddl/ob_ddl_merge_task.cpp +++ b/src/storage/ddl/ob_ddl_merge_task.cpp @@ -416,9 +416,6 @@ int ObDDLTableMergeTask::merge_full_direct_load_ddl_kvs(ObLSHandle &ls_handle, O } else if (nullptr != first_major_sstable) { is_major_exist = true; LOG_INFO("major sstable has been created before", K(merge_param_)); - } else if (tablet.get_tablet_meta().table_store_flag_.with_major_sstable()) { - ret = OB_TASK_EXPIRED; - LOG_INFO("tablet me says with major but no major, meaning its a migrated deleted tablet, skip"); } else if (OB_FAIL(tenant_direct_load_mgr->get_tablet_mgr(merge_param_.tablet_id_, true /* is_full_direct_load */, tablet_mgr_hdl))) { diff --git a/src/storage/ddl/ob_ddl_redo_log_writer.cpp b/src/storage/ddl/ob_ddl_redo_log_writer.cpp index c8db718965..115246397f 100644 --- a/src/storage/ddl/ob_ddl_redo_log_writer.cpp +++ b/src/storage/ddl/ob_ddl_redo_log_writer.cpp @@ -1492,6 +1492,7 @@ int ObDDLRedoLogWriter::remote_write_ddl_commit_redo(const obrpc::ObRpcRemoteWri ObDDLRedoLogWriter::~ObDDLRedoLogWriter() { + ddl_redo_handle_.reset(); if (nullptr != buffer_) { ob_free(buffer_); buffer_ = nullptr; diff --git a/src/storage/ddl/ob_ddl_struct.cpp b/src/storage/ddl/ob_ddl_struct.cpp index c705f9b466..2b20d93f9e 100644 --- a/src/storage/ddl/ob_ddl_struct.cpp +++ b/src/storage/ddl/ob_ddl_struct.cpp @@ -342,7 +342,7 @@ bool ObDDLMacroBlockRedoInfo::is_valid() const { return table_key_.is_valid() && data_buffer_.ptr() != nullptr && block_type_ != ObDDLMacroBlockType::DDL_MB_INVALID_TYPE && logic_id_.is_valid() && start_scn_.is_valid_and_not_min() && data_format_version_ >= 0 - && type_ > ObDirectLoadType::DIRECT_LOAD_INVALID && type_ < ObDirectLoadType::DIRECT_LOAD_MAX + && type_ < ObDirectLoadType::DIRECT_LOAD_MAX && (is_incremental_direct_load(type_) ? trans_id_.is_valid() : !trans_id_.is_valid()); } diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp index baf3d185a0..c1bd5a78e9 100644 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp @@ -2250,7 +2250,11 @@ void ObTabletDirectLoadMgr::unlock(const uint32_t tid) int ObTabletDirectLoadMgr::prepare_storage_schema(ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; - sqc_build_ctx_.storage_schema_ = nullptr; + if (nullptr != sqc_build_ctx_.storage_schema_) { + ObTabletObjLoadHelper::free(sqc_build_ctx_.schema_allocator_, sqc_build_ctx_.storage_schema_); + sqc_build_ctx_.storage_schema_ = nullptr; + sqc_build_ctx_.schema_allocator_.reset(); + } if (OB_UNLIKELY(!tablet_handle.is_valid())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid tablet handle", K(ret), K(tablet_handle)); diff --git a/src/storage/high_availability/ob_ls_complete_migration.cpp b/src/storage/high_availability/ob_ls_complete_migration.cpp index 3a80d47490..ab8b74bbcb 100644 --- a/src/storage/high_availability/ob_ls_complete_migration.cpp +++ b/src/storage/high_availability/ob_ls_complete_migration.cpp @@ -177,6 +177,7 @@ int ObLSCompleteMigrationDagNet::start_running_for_migration_() int tmp_ret = OB_SUCCESS; ObInitialCompleteMigrationDag *initial_dag = nullptr; ObTenantDagScheduler *scheduler = nullptr; + ObDagPrio::ObDagPrioEnum prio = ObDagPrio::DAG_PRIO_MAX; if (!is_inited_) { ret = OB_NOT_INIT; @@ -185,8 +186,10 @@ int ObLSCompleteMigrationDagNet::start_running_for_migration_() } else if (OB_ISNULL(scheduler = MTL(ObTenantDagScheduler*))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to get ObTenantDagScheduler from MTL", K(ret)); - } else if (OB_FAIL(scheduler->alloc_dag(initial_dag))) { - LOG_WARN("failed to alloc initial dag ", K(ret)); + } else if (OB_FAIL(ObMigrationUtils::get_dag_priority(ctx_.arg_.type_, prio))) { + LOG_WARN("failed to get dag priority", K(ret)); + } else if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, initial_dag))) { + LOG_WARN("failed to alloc initial dag ", K(ret), K(prio)); } else if (OB_FAIL(initial_dag->init(this))) { LOG_WARN("failed to init initial dag", K(ret)); } else if (OB_FAIL(add_dag_into_dag_net(*initial_dag))) { @@ -663,8 +666,9 @@ int ObInitialCompleteMigrationDag::fill_dag_key(char *buf, const int64_t buf_len LOG_WARN("ha dag net ctx type is unexpected", K(ret), KPC(ha_dag_net_ctx_)); } else if (FALSE_IT(self_ctx = static_cast(ha_dag_net_ctx_))) { } else if (OB_FAIL(databuff_printf(buf, buf_len, - "ObInitialCompleteMigrationDag: ls_id = %s, migration_type = %s", - to_cstring(self_ctx->arg_.ls_id_), ObMigrationOpType::get_str(self_ctx->arg_.type_)))) { + "ObInitialCompleteMigrationDag: ls_id = %s, migration_type = %s, dag_prio = %s", + to_cstring(self_ctx->arg_.ls_id_), ObMigrationOpType::get_str(self_ctx->arg_.type_), + ObIDag::get_dag_prio_str(this->get_priority())))) { LOG_WARN("failed to fill comment", K(ret), K(*self_ctx)); } return ret; @@ -798,6 +802,7 @@ int ObInitialCompleteMigrationTask::generate_migration_dags_() ObFinishCompleteMigrationDag *finish_complete_dag = nullptr; ObTenantDagScheduler *scheduler = nullptr; ObInitialCompleteMigrationDag *initial_complete_migration_dag = nullptr; + ObDagPrio::ObDagPrioEnum prio = ObDagPrio::DAG_PRIO_MAX; if (!is_inited_) { ret = OB_NOT_INIT; @@ -808,10 +813,12 @@ int ObInitialCompleteMigrationTask::generate_migration_dags_() } else if (OB_ISNULL(initial_complete_migration_dag = static_cast(this->get_dag()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("initial complete migration dag should not be NULL", K(ret), KP(initial_complete_migration_dag)); + } else if (OB_FAIL(ObMigrationUtils::get_dag_priority(ctx_->arg_.type_, prio))) { + LOG_WARN("failed to get dag priority", K(ret)); } else { - if (OB_FAIL(scheduler->alloc_dag(start_complete_dag))) { + if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, start_complete_dag))) { LOG_WARN("failed to alloc start complete migration dag ", K(ret)); - } else if (OB_FAIL(scheduler->alloc_dag(finish_complete_dag))) { + } else if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, finish_complete_dag))) { LOG_WARN("failed to alloc finish complete migration dag", K(ret)); } else if (OB_FAIL(start_complete_dag->init(dag_net_))) { LOG_WARN("failed to init start complete migration dag", K(ret)); @@ -892,8 +899,9 @@ int ObStartCompleteMigrationDag::fill_dag_key(char *buf, const int64_t buf_len) LOG_WARN("ha dag net ctx type is unexpected", K(ret), KPC(ha_dag_net_ctx_)); } else if (FALSE_IT(self_ctx = static_cast(ha_dag_net_ctx_))) { } else if (OB_FAIL(databuff_printf(buf, buf_len, - "ObStartPrepareMigrationDag: ls_id = %s, migration_type = %s", - to_cstring(self_ctx->arg_.ls_id_), ObMigrationOpType::get_str(self_ctx->arg_.type_)))) { + "ObStartPrepareMigrationDag: ls_id = %s, migration_type = %s, dag_prio = %s", + to_cstring(self_ctx->arg_.ls_id_), ObMigrationOpType::get_str(self_ctx->arg_.type_), + ObIDag::get_dag_prio_str(this->get_priority())))) { LOG_WARN("failed to fill comment", K(ret), K(*self_ctx)); } return ret; @@ -1459,6 +1467,8 @@ int ObStartCompleteMigrationTask::change_member_list_() const int64_t cost_ts = ObTimeUtility::current_time() - start_ts; LOG_INFO("succeed change member list", "cost", cost_ts, "tenant_id", ctx_->tenant_id_, "ls_id", ctx_->arg_.ls_id_); } + + DEBUG_SYNC(AFTER_MEMBERLIST_CHANGED); return ret; } @@ -2092,8 +2102,9 @@ int ObFinishCompleteMigrationDag::fill_dag_key(char *buf, const int64_t buf_len) LOG_WARN("ha dag net ctx type is unexpected", K(ret), KPC(ha_dag_net_ctx_)); } else if (FALSE_IT(self_ctx = static_cast(ha_dag_net_ctx_))) { } else if (OB_FAIL(databuff_printf(buf, buf_len, - "ObFinishCompleteMigrationDag: ls_id = %s, migration_type = %s", - to_cstring(self_ctx->arg_.ls_id_), ObMigrationOpType::get_str(self_ctx->arg_.type_)))) { + "ObFinishCompleteMigrationDag: ls_id = %s, migration_type = %s, dag_prio = %s", + to_cstring(self_ctx->arg_.ls_id_), ObMigrationOpType::get_str(self_ctx->arg_.type_), + ObIDag::get_dag_prio_str(this->get_priority())))) { LOG_WARN("failed to fill comment", K(ret), K(*self_ctx)); } return ret; @@ -2220,6 +2231,7 @@ int ObFinishCompleteMigrationTask::generate_prepare_initial_dag_() ObInitialCompleteMigrationDag *initial_complete_dag = nullptr; ObTenantDagScheduler *scheduler = nullptr; ObFinishCompleteMigrationDag *finish_complete_migration_dag = nullptr; + ObDagPrio::ObDagPrioEnum prio = ObDagPrio::DAG_PRIO_MAX; if (!is_inited_) { ret = OB_NOT_INIT; @@ -2230,8 +2242,10 @@ int ObFinishCompleteMigrationTask::generate_prepare_initial_dag_() } else if (OB_ISNULL(scheduler = MTL(ObTenantDagScheduler*))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to get ObTenantDagScheduler from MTL", K(ret)); + } else if (OB_FAIL(ObMigrationUtils::get_dag_priority(ctx_->arg_.type_, prio))) { + LOG_WARN("failed to get dag priority", K(ret)); } else { - if (OB_FAIL(scheduler->alloc_dag(initial_complete_dag))) { + if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, initial_complete_dag))) { LOG_WARN("failed to alloc initial complete migration dag ", K(ret)); } else if (OB_FAIL(initial_complete_dag->init(dag_net_))) { LOG_WARN("failed to init initial complete migration dag", K(ret)); diff --git a/src/storage/high_availability/ob_ls_migration.cpp b/src/storage/high_availability/ob_ls_migration.cpp index bc4a2b70e1..205496d4ff 100644 --- a/src/storage/high_availability/ob_ls_migration.cpp +++ b/src/storage/high_availability/ob_ls_migration.cpp @@ -311,6 +311,7 @@ int ObMigrationDagNet::start_running_for_migration_() int tmp_ret = OB_SUCCESS; ObInitialMigrationDag *initial_migration_dag = nullptr; ObTenantDagScheduler *scheduler = nullptr; + ObDagPrio::ObDagPrioEnum prio = ObDagPrio::DAG_PRIO_MAX; if (!is_inited_) { ret = OB_NOT_INIT; @@ -319,8 +320,10 @@ int ObMigrationDagNet::start_running_for_migration_() } else if (OB_ISNULL(scheduler = MTL(ObTenantDagScheduler*))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to get ObTenantDagScheduler from MTL", K(ret)); - } else if (OB_FAIL(scheduler->alloc_dag(initial_migration_dag))) { - LOG_WARN("failed to alloc migration dag ", K(ret)); + } else if (OB_FAIL(ObMigrationUtils::get_dag_priority(ctx_->arg_.type_, prio))) { + LOG_WARN("failed to get dag priority", K(ret)); + } else if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, initial_migration_dag))) { + LOG_WARN("failed to alloc migration dag ", K(ret), K(prio)); } else if (OB_FAIL(initial_migration_dag->init(this))) { LOG_WARN("failed to init migration dag", K(ret)); } else if (OB_FAIL(add_dag_into_dag_net(*initial_migration_dag))) { @@ -549,8 +552,9 @@ int ObInitialMigrationDag::fill_dag_key(char *buf, const int64_t buf_len) const ret = OB_ERR_UNEXPECTED; LOG_WARN("inital migration dag migration ctx should not be NULL", K(ret), KP(ctx)); } else if (OB_FAIL(databuff_printf(buf, buf_len, - "ObInitialMigrationDag: ls_id = %s, migration_type = %s", - to_cstring(ctx->arg_.ls_id_), ObMigrationOpType::get_str(ctx->arg_.type_)))) { + "ObInitialMigrationDag: ls_id = %s, migration_type = %s, dag_prio = %s", + to_cstring(ctx->arg_.ls_id_), ObMigrationOpType::get_str(ctx->arg_.type_), + ObIDag::get_dag_prio_str(this->get_priority())))) { LOG_WARN("failed to fill comment", K(ret), KPC(ctx)); } return ret; @@ -703,6 +707,7 @@ int ObInitialMigrationTask::generate_migration_dags_() ObMigrationFinishDag *migration_finish_dag = nullptr; ObTenantDagScheduler *scheduler = nullptr; ObInitialMigrationDag *initial_migration_dag = nullptr; + ObDagPrio::ObDagPrioEnum prio = ObDagPrio::DAG_PRIO_MAX; if (!is_inited_) { ret = OB_NOT_INIT; @@ -713,9 +718,11 @@ int ObInitialMigrationTask::generate_migration_dags_() } else if (OB_ISNULL(initial_migration_dag = static_cast(this->get_dag()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("initial migration dag is null", K(ret), KP(initial_migration_dag)); - } else if (OB_FAIL(scheduler->alloc_dag(start_migration_dag))) { + } else if (OB_FAIL(ObMigrationUtils::get_dag_priority(ctx_->arg_.type_, prio))) { + LOG_WARN("failed to get dag priority", K(ret)); + } else if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, start_migration_dag))) { LOG_WARN("failed to alloc start migration dag ", K(ret)); - } else if (OB_FAIL(scheduler->alloc_dag(migration_finish_dag))) { + } else if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, migration_finish_dag))) { LOG_WARN("failed to alloc migration finish dag", K(ret)); } else if (OB_FAIL(start_migration_dag->init(dag_net_))) { LOG_WARN("failed to init start migration dag", K(ret)); @@ -845,8 +852,9 @@ int ObStartMigrationDag::fill_dag_key(char *buf, const int64_t buf_len) const ret = OB_ERR_UNEXPECTED; LOG_WARN("migration ctx should not be NULL", K(ret), KP(ctx)); } else if (OB_FAIL(databuff_printf(buf, buf_len, - "ObStartMigrationDag: ls_id = %s, migration_type = %s", - to_cstring(ctx->arg_.ls_id_), ObMigrationOpType::get_str(ctx->arg_.type_)))) { + "ObStartMigrationDag: ls_id = %s, migration_type = %s, dag_prio = %s", + to_cstring(ctx->arg_.ls_id_), ObMigrationOpType::get_str(ctx->arg_.type_), + ObIDag::get_dag_prio_str(this->get_priority())))) { LOG_WARN("failed to fill comment", K(ret), KPC(ctx)); } return ret; @@ -1278,6 +1286,7 @@ int ObStartMigrationTask::generate_tablets_migration_dag_() ObTenantDagScheduler *scheduler = nullptr; ObIDagNet *dag_net = nullptr; ObStartMigrationDag *start_migration_dag = nullptr; + ObDagPrio::ObDagPrioEnum prio = ObDagPrio::DAG_PRIO_MAX; if (!is_inited_) { ret = OB_NOT_INIT; @@ -1291,10 +1300,12 @@ int ObStartMigrationTask::generate_tablets_migration_dag_() } else if (OB_ISNULL(scheduler = MTL(ObTenantDagScheduler*))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to get ObTenantDagScheduler from MTL", K(ret)); + } else if (OB_FAIL(ObMigrationUtils::get_dag_priority(ctx_->arg_.type_, prio))) { + LOG_WARN("failed to get dag priority", K(ret)); } else { - if (OB_FAIL(scheduler->alloc_dag(sys_tablets_migration_dag))) { + if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, sys_tablets_migration_dag))) { LOG_WARN("failed to alloc sys tablets migration dag ", K(ret)); - } else if (OB_FAIL(scheduler->alloc_dag(data_tablets_migration_dag))) { + } else if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, data_tablets_migration_dag))) { LOG_WARN("failed to alloc data tablets migration dag ", K(ret)); } else if (OB_FAIL(sys_tablets_migration_dag->init(dag_net))) { LOG_WARN("failed to init sys tablets migration dag", K(ret), K(*ctx_)); @@ -1697,8 +1708,9 @@ int ObSysTabletsMigrationDag::fill_dag_key(char *buf, const int64_t buf_len) con ret = OB_ERR_UNEXPECTED; LOG_WARN("migration ctx should not be NULL", K(ret), KP(ctx)); } else if (OB_FAIL(databuff_printf(buf, buf_len, - "ObSysTabletsMigrationDag: ls_id = %s, migration_type = %s", - to_cstring(ctx->arg_.ls_id_), ObMigrationOpType::get_str(ctx->arg_.type_)))) { + "ObSysTabletsMigrationDag: ls_id = %s, migration_type = %s, dag_prio = %s", + to_cstring(ctx->arg_.ls_id_), ObMigrationOpType::get_str(ctx->arg_.type_), + ObIDag::get_dag_prio_str(this->get_priority())))) { LOG_WARN("failed to fill comment", K(ret), KPC(ctx)); } return ret; @@ -1866,6 +1878,7 @@ int ObSysTabletsMigrationTask::generate_sys_tablet_migartion_dag_() ObSysTabletsMigrationDag *sys_tablets_migration_dag = nullptr; ObLS *ls = nullptr; ObIDag *parent = nullptr; + ObDagPrio::ObDagPrioEnum prio = ObDagPrio::DAG_PRIO_MAX; if (!is_inited_) { ret = OB_NOT_INIT; @@ -1885,6 +1898,8 @@ int ObSysTabletsMigrationTask::generate_sys_tablet_migartion_dag_() } else if (OB_FALSE_IT(parent = this->get_dag())) { } else if (OB_FAIL(tablet_migration_dag_array.push_back(parent))) { LOG_WARN("failed to push sys_tablets_migration_dag into array", K(ret), K(*ctx_)); + } else if (OB_FAIL(ObMigrationUtils::get_dag_priority(ctx_->arg_.type_, prio))) { + LOG_WARN("failed to get dag priority", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < ctx_->sys_tablet_id_array_.count(); ++i) { const ObTabletID &tablet_id = ctx_->sys_tablet_id_array_.at(i); @@ -1892,7 +1907,7 @@ int ObSysTabletsMigrationTask::generate_sys_tablet_migartion_dag_() ObTabletHandle tablet_handle; if (OB_FAIL(ls->ha_get_tablet(tablet_id, tablet_handle))) { LOG_WARN("failed to get tablet", K(ret), K(tablet_id)); - } else if (OB_FAIL(scheduler->alloc_dag(tablet_migration_dag))) { + } else if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, tablet_migration_dag))) { LOG_WARN("failed to alloc tablet migration dag ", K(ret)); } else if (OB_FAIL(tablet_migration_dag_array.push_back(tablet_migration_dag))) { LOG_WARN("failed to push tablet migration dag into array", K(ret), K(*ctx_)); @@ -2033,8 +2048,9 @@ int ObTabletMigrationDag::fill_dag_key(char *buf, const int64_t buf_len) const ret = OB_ERR_UNEXPECTED; LOG_WARN("migration ctx should not be NULL", K(ret), KP(ctx)); } else if (OB_FAIL(databuff_printf(buf, buf_len, - "ObTabletMigrationDag: log_stream_id = %s, tablet_id = %s, migration_type = %s", - to_cstring(ctx->arg_.ls_id_), to_cstring(copy_tablet_ctx_.tablet_id_), ObMigrationOpType::get_str(ctx->arg_.type_)))) { + "ObTabletMigrationDag: log_stream_id = %s, tablet_id = %s, migration_type = %s, dag_prio = %s", + to_cstring(ctx->arg_.ls_id_), to_cstring(copy_tablet_ctx_.tablet_id_), ObMigrationOpType::get_str(ctx->arg_.type_), + ObIDag::get_dag_prio_str(this->get_priority())))) { LOG_WARN("failed to fill comment", K(ret), KPC(ctx), K(copy_tablet_ctx_)); } @@ -2156,6 +2172,7 @@ int ObTabletMigrationDag::generate_next_dag(share::ObIDag *&dag) const int64_t start_ts = ObTimeUtil::current_time(); ObMigrationCtx *ctx = nullptr; ObLS *ls = nullptr; + ObDagPrio::ObDagPrioEnum prio = ObDagPrio::DAG_PRIO_MAX; if (!is_inited_) { ret = OB_NOT_INIT; @@ -2175,6 +2192,8 @@ int ObTabletMigrationDag::generate_next_dag(share::ObIDag *&dag) } else if (OB_ISNULL(ls = ls_handle_.get_ls())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls should not be NULL", K(ret), KPC(ctx)); + } else if (OB_FAIL(ObMigrationUtils::get_dag_priority(ctx->arg_.type_, prio))) { + LOG_WARN("failed to get dag priority", K(ret)); } else { while (OB_SUCC(ret)) { ObTabletHandle tablet_handle; @@ -2197,7 +2216,7 @@ int ObTabletMigrationDag::generate_next_dag(share::ObIDag *&dag) } else { LOG_WARN("failed to get tablet", K(ret), K(tablet_id)); } - } else if (OB_FAIL(scheduler->alloc_dag(tablet_migration_dag))) { + } else if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, tablet_migration_dag))) { LOG_WARN("failed to alloc tablet migration dag", K(ret)); } else { if (OB_FAIL(tablet_migration_dag->init(tablet_id, tablet_handle, dag_net, tablet_group_ctx_))) { @@ -3139,8 +3158,9 @@ int ObDataTabletsMigrationDag::fill_dag_key(char *buf, const int64_t buf_len) co ret = OB_ERR_UNEXPECTED; LOG_WARN("migration ctx should not be NULL", K(ret), KP(ctx)); } else if (OB_FAIL(databuff_printf(buf, buf_len, - "ObDataTabletsMigrationDag: log_stream_id = %s, migration_type = %s", - to_cstring(ctx->arg_.ls_id_), ObMigrationOpType::get_str(ctx->arg_.type_)))) { + "ObDataTabletsMigrationDag: log_stream_id = %s, migration_type = %s, dag_prio = %s", + to_cstring(ctx->arg_.ls_id_), ObMigrationOpType::get_str(ctx->arg_.type_), + ObIDag::get_dag_prio_str(this->get_priority())))) { LOG_WARN("failed to fill comment", K(ret), KPC(ctx)); } return ret; @@ -3551,6 +3571,7 @@ int ObDataTabletsMigrationTask::generate_tablet_group_dag_() ObDataTabletsMigrationDag *data_tablets_migration_dag = nullptr; ObHATabletGroupCtx *tablet_group_ctx = nullptr; ObArray tablet_id_array; + ObDagPrio::ObDagPrioEnum prio = ObDagPrio::DAG_PRIO_MAX; DEBUG_SYNC(BEFORE_TABLET_GROUP_MIGRATION_GENERATE_NEXT_DAG); @@ -3574,8 +3595,10 @@ int ObDataTabletsMigrationTask::generate_tablet_group_dag_() } else if (OB_ISNULL(scheduler = MTL(ObTenantDagScheduler*))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to get ObTenantDagScheduler from MTL", K(ret)); + } else if (OB_FAIL(ObMigrationUtils::get_dag_priority(ctx_->arg_.type_, prio))) { + LOG_WARN("failed to get dag priority", K(ret)); } else { - if (OB_FAIL(scheduler->alloc_dag(tablet_group_dag))) { + if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, tablet_group_dag))) { LOG_WARN("failed to alloc tablet group migration dag ", K(ret)); } else if (OB_FAIL(tablet_group_dag->init(tablet_id_array, dag_net, finish_dag_, tablet_group_ctx))) { LOG_WARN("failed to init tablet group dag", K(ret), K(tablet_id_array)); @@ -3798,9 +3821,9 @@ int ObTabletGroupMigrationDag::fill_dag_key(char *buf, const int64_t buf_len) co ret = OB_ERR_UNEXPECTED; LOG_WARN("migration ctx should not be NULL", K(ret), KP(ctx)); } else if (OB_FAIL(databuff_printf(buf, buf_len, - "ObTabletGroupMigrationDag: log_stream_id = %s, migration_type = %s, first_tablet_id = %s", + "ObTabletGroupMigrationDag: log_stream_id = %s, migration_type = %s, first_tablet_id = %s, dag_prio = %s", to_cstring(ctx->arg_.ls_id_), ObMigrationOpType::get_str(ctx->arg_.type_), - to_cstring(tablet_id_array_.at(0))))) { + to_cstring(tablet_id_array_.at(0)), ObIDag::get_dag_prio_str(this->get_priority())))) { LOG_WARN("failed to fill comment", K(ret), KPC(ctx)); } return ret; @@ -3872,6 +3895,7 @@ int ObTabletGroupMigrationDag::generate_next_dag(share::ObIDag *&dag) ObArray tablet_id_array; ObDagId dag_id; const int64_t start_ts = ObTimeUtil::current_time(); + ObDagPrio::ObDagPrioEnum prio = ObDagPrio::DAG_PRIO_MAX; if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("tablet group migration dag do not init", K(ret)); @@ -3899,7 +3923,9 @@ int ObTabletGroupMigrationDag::generate_next_dag(share::ObIDag *&dag) } else if (OB_ISNULL(scheduler = MTL(ObTenantDagScheduler*))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to get ObTenantDagScheduler from MTL", K(ret)); - } else if (OB_FAIL(scheduler->alloc_dag(tablet_group_migration_dag))) { + } else if (OB_FAIL(ObMigrationUtils::get_dag_priority(ctx->arg_.type_, prio))) { + LOG_WARN("failed to get dag priority", K(ret)); + } else if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, tablet_group_migration_dag))) { LOG_WARN("failed to alloc tablet backfill tx migration dag ", K(ret)); } else if (OB_FAIL(tablet_group_migration_dag->init(tablet_id_array, dag_net, finish_dag_, tablet_group_ctx))) { LOG_WARN("failed to init tablet migration dag", K(ret), KPC(ctx)); @@ -4075,6 +4101,7 @@ int ObTabletGroupMigrationTask::generate_tablet_migration_dag_() ObTabletGroupMigrationDag *tablet_group_migration_dag = nullptr; ObTabletMigrationDag *tablet_migration_dag = nullptr; ObLS *ls = nullptr; + ObDagPrio::ObDagPrioEnum prio = ObDagPrio::DAG_PRIO_MAX; DEBUG_SYNC(BEFORE_TABLET_MIGRATION_GENERATE_NEXT_DAG); @@ -4093,6 +4120,8 @@ int ObTabletGroupMigrationTask::generate_tablet_migration_dag_() } else if (OB_ISNULL(ls = ls_handle_.get_ls())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls should not be NULL", K(ret), KP(ls)); + } else if (OB_FAIL(ObMigrationUtils::get_dag_priority(ctx_->arg_.type_, prio))) { + LOG_WARN("failed to get dag priority", K(ret)); } else { ObIDag *parent = this->get_dag(); ObTabletID tablet_id; @@ -4112,7 +4141,7 @@ int ObTabletGroupMigrationTask::generate_tablet_migration_dag_() } else { LOG_WARN("failed to get tablet", K(ret), K(tablet_id)); } - } else if (OB_FAIL(scheduler->alloc_dag(tablet_migration_dag))) { + } else if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, tablet_migration_dag))) { LOG_WARN("failed to alloc tablet migration dag ", K(ret)); } else if (OB_FAIL(tablet_migration_dag->init(tablet_id, tablet_handle, dag_net, tablet_group_ctx_))) { LOG_WARN("failed to init tablet migration migration dag", K(ret), K(*ctx_)); @@ -4287,8 +4316,9 @@ int ObMigrationFinishDag::fill_dag_key(char *buf, const int64_t buf_len) const ret = OB_ERR_UNEXPECTED; LOG_WARN("migration ctx should not be NULL", K(ret), KP(ctx)); } else if (OB_FAIL(databuff_printf(buf, buf_len, - "ObMigrationFinishDag: ls_id = %s, migration_type = %s", - to_cstring(ctx->arg_.ls_id_), ObMigrationOpType::get_str(ctx->arg_.type_)))) { + "ObMigrationFinishDag: ls_id = %s, migration_type = %s, dag_prio = %s", + to_cstring(ctx->arg_.ls_id_), ObMigrationOpType::get_str(ctx->arg_.type_), + ObIDag::get_dag_prio_str(this->get_priority())))) { LOG_WARN("failed to fill comment", K(ret), KPC(ctx)); } return ret; @@ -4412,6 +4442,7 @@ int ObMigrationFinishTask::generate_migration_init_dag_() ObInitialMigrationDag *initial_migration_dag = nullptr; ObTenantDagScheduler *scheduler = nullptr; ObMigrationFinishDag *migration_finish_dag = nullptr; + ObDagPrio::ObDagPrioEnum prio = ObDagPrio::DAG_PRIO_MAX; if (!is_inited_) { ret = OB_NOT_INIT; @@ -4422,8 +4453,10 @@ int ObMigrationFinishTask::generate_migration_init_dag_() } else if (OB_ISNULL(migration_finish_dag = static_cast(this->get_dag()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("migration finish dag should not be NULL", K(ret), KP(migration_finish_dag)); + } else if (OB_FAIL(ObMigrationUtils::get_dag_priority(ctx_->arg_.type_, prio))) { + LOG_WARN("failed to get dag priority", K(ret)); } else { - if (OB_FAIL(scheduler->alloc_dag(initial_migration_dag))) { + if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, initial_migration_dag))) { LOG_WARN("failed to alloc initial migration dag ", K(ret)); } else if (OB_FAIL(initial_migration_dag->init(dag_net_))) { LOG_WARN("failed to init initial migration dag", K(ret)); diff --git a/src/storage/high_availability/ob_ls_prepare_migration.cpp b/src/storage/high_availability/ob_ls_prepare_migration.cpp index f69278b329..8efc69a529 100644 --- a/src/storage/high_availability/ob_ls_prepare_migration.cpp +++ b/src/storage/high_availability/ob_ls_prepare_migration.cpp @@ -156,6 +156,7 @@ int ObLSPrepareMigrationDagNet::start_running_for_migration_() int tmp_ret = OB_SUCCESS; ObInitialPrepareMigrationDag *initial_dag = nullptr; ObTenantDagScheduler *scheduler = nullptr; + ObDagPrio::ObDagPrioEnum prio = ObDagPrio::DAG_PRIO_MAX; if (!is_inited_) { ret = OB_NOT_INIT; @@ -164,7 +165,9 @@ int ObLSPrepareMigrationDagNet::start_running_for_migration_() } else if (OB_ISNULL(scheduler = MTL(ObTenantDagScheduler*))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to get ObTenantDagScheduler from MTL", K(ret)); - } else if (OB_FAIL(scheduler->alloc_dag(initial_dag))) { + } else if (OB_FAIL(ObMigrationUtils::get_dag_priority(ctx_.arg_.type_, prio))) { + LOG_WARN("failed to get dag priority", K(ret)); + } else if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, initial_dag))) { LOG_WARN("failed to alloc initial dag ", K(ret)); } else if (OB_FAIL(initial_dag->init(this))) { LOG_WARN("failed to init initial dag", K(ret)); @@ -425,8 +428,9 @@ int ObInitialPrepareMigrationDag::fill_dag_key(char *buf, const int64_t buf_len) LOG_WARN("ha dag net ctx type is unexpected", K(ret), KPC(ha_dag_net_ctx_)); } else if (FALSE_IT(self_ctx = static_cast(ha_dag_net_ctx_))) { } else if (OB_FAIL(databuff_printf(buf, buf_len, - "ObInitialPrepareMigrationDag: ls_id = %s, migration_type = %s", - to_cstring(self_ctx->arg_.ls_id_), ObMigrationOpType::get_str(self_ctx->arg_.type_)))) { + "ObInitialPrepareMigrationDag: ls_id = %s, migration_type = %s, dag_prio = %s", + to_cstring(self_ctx->arg_.ls_id_), ObMigrationOpType::get_str(self_ctx->arg_.type_), + ObIDag::get_dag_prio_str(this->get_priority())))) { LOG_WARN("failed to fill comment", K(ret), K(*self_ctx)); } return ret; @@ -537,6 +541,7 @@ int ObInitialPrepareMigrationTask::generate_migration_dags_() ObFinishPrepareMigrationDag *finish_prepare_dag = nullptr; ObTenantDagScheduler *scheduler = nullptr; ObInitialPrepareMigrationDag *initial_prepare_migration_dag = nullptr; + ObDagPrio::ObDagPrioEnum prio = ObDagPrio::DAG_PRIO_MAX; if (!is_inited_) { ret = OB_NOT_INIT; @@ -547,10 +552,12 @@ int ObInitialPrepareMigrationTask::generate_migration_dags_() } else if (OB_ISNULL(initial_prepare_migration_dag = static_cast(this->get_dag()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("initial prepare migration dag should not be NULL", K(ret), KP(initial_prepare_migration_dag)); + } else if (OB_FAIL(ObMigrationUtils::get_dag_priority(ctx_->arg_.type_, prio))) { + LOG_WARN("failed to get dag priority", K(ret)); } else { - if (OB_FAIL(scheduler->alloc_dag(start_prepare_dag))) { + if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, start_prepare_dag))) { LOG_WARN("failed to alloc start prepare migration dag ", K(ret)); - } else if (OB_FAIL(scheduler->alloc_dag(finish_prepare_dag))) { + } else if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, finish_prepare_dag))) { LOG_WARN("failed to alloc finish prepare migration dag", K(ret)); } else if (OB_FAIL(start_prepare_dag->init(dag_net_))) { LOG_WARN("failed to init start prepare migration dag", K(ret)); @@ -630,8 +637,9 @@ int ObStartPrepareMigrationDag::fill_dag_key(char *buf, const int64_t buf_len) c LOG_WARN("ha dag net ctx type is unexpected", K(ret), KPC(ha_dag_net_ctx_)); } else if (FALSE_IT(self_ctx = static_cast(ha_dag_net_ctx_))) { } else if (OB_FAIL(databuff_printf(buf, buf_len, - "ObStartPrepareMigrationDag: ls_id = %s, migration_type = %s", - to_cstring(self_ctx->arg_.ls_id_), ObMigrationOpType::get_str(self_ctx->arg_.type_)))) { + "ObStartPrepareMigrationDag: ls_id = %s, migration_type = %s, dag_prio = %s", + to_cstring(self_ctx->arg_.ls_id_), ObMigrationOpType::get_str(self_ctx->arg_.type_), + ObIDag::get_dag_prio_str(this->get_priority())))) { LOG_WARN("failed to fill comment", K(ret), K(*self_ctx)); } return ret; @@ -973,6 +981,7 @@ int ObStartPrepareMigrationTask::generate_prepare_migration_dags_() ObLS *ls = nullptr; common::ObArray tablet_infos; storage::ObTabletBackfillInfo tablet_info; + ObDagPrio::ObDagPrioEnum prio = ObDagPrio::DAG_PRIO_MAX; if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("start prepare migration task do not init", K(ret)); @@ -992,8 +1001,10 @@ int ObStartPrepareMigrationTask::generate_prepare_migration_dags_() } else if (OB_ISNULL(scheduler = MTL(ObTenantDagScheduler*))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to get ObTenantDagScheduler from MTL", K(ret)); + } else if (OB_FAIL(ObMigrationUtils::get_dag_priority(ctx_->arg_.type_, prio))) { + LOG_WARN("failed to get dag priority", K(ret)); } else { - if (OB_FAIL(scheduler->alloc_dag(finish_backfill_tx_dag))) { + if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, finish_backfill_tx_dag))) { LOG_WARN("failed to alloc finish backfill tx migration dag ", K(ret)); } else if (OB_FAIL(build_tablet_backfill_info_(tablet_infos))) { LOG_WARN("failed to build tablet backfill info", K(ret)); @@ -1009,7 +1020,7 @@ int ObStartPrepareMigrationTask::generate_prepare_migration_dags_() } else { if (OB_FAIL(backfill_tx_ctx->get_tablet_info(tablet_info))) { LOG_WARN("failed to get tablet id", K(ret), KPC(ctx_)); - } else if (OB_FAIL(scheduler->alloc_dag(tablet_backfill_tx_dag))) { + } else if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, tablet_backfill_tx_dag))) { LOG_WARN("failed to alloc tablet backfill tx migration dag ", K(ret)); } else if (OB_FAIL(tablet_backfill_tx_dag->init(ctx_->task_id_, ctx_->arg_.ls_id_, tablet_info, ctx_, backfill_tx_ctx))) { LOG_WARN("failed to init tablet backfill tx dag", K(ret), K(*ctx_)); @@ -1321,7 +1332,8 @@ int ObStartPrepareMigrationTask::wait_transfer_out_tablet_ready_( } else if (ObMigrationStatus::OB_MIGRATION_STATUS_NONE != status && ObMigrationStatus::OB_MIGRATION_STATUS_REBUILD_WAIT != status && ObMigrationStatus::OB_MIGRATION_STATUS_MIGRATE_WAIT != status - && ObMigrationStatus::OB_MIGRATION_STATUS_ADD_WAIT != status) { + && ObMigrationStatus::OB_MIGRATION_STATUS_ADD_WAIT != status + && ObMigrationStatus::OB_MIGRATION_STATUS_HOLD != status) { if (ObMigrationStatus::OB_MIGRATION_STATUS_ADD_FAIL == status || ObMigrationStatus::OB_MIGRATION_STATUS_MIGRATE_FAIL == status || ObMigrationStatus::OB_MIGRATION_STATUS_REBUILD_FAIL == status) { @@ -1407,8 +1419,9 @@ int ObFinishPrepareMigrationDag::fill_dag_key(char *buf, const int64_t buf_len) LOG_WARN("ha dag net ctx type is unexpected", K(ret), KPC(ha_dag_net_ctx_)); } else if (FALSE_IT(self_ctx = static_cast(ha_dag_net_ctx_))) { } else if (OB_FAIL(databuff_printf(buf, buf_len, - "ObFinishPrepareMigrationDag: ls_id = %s, migration_type = %s", - to_cstring(self_ctx->arg_.ls_id_), ObMigrationOpType::get_str(self_ctx->arg_.type_)))) { + "ObFinishPrepareMigrationDag: ls_id = %s, migration_type = %s, dag_prio = %s", + to_cstring(self_ctx->arg_.ls_id_), ObMigrationOpType::get_str(self_ctx->arg_.type_), + ObIDag::get_dag_prio_str(this->get_priority())))) { LOG_WARN("failed to fill comment", K(ret), K(*self_ctx)); } return ret; @@ -1523,6 +1536,7 @@ int ObFinishPrepareMigrationTask::generate_prepare_initial_dag_() ObInitialPrepareMigrationDag *initial_prepare_dag = nullptr; ObTenantDagScheduler *scheduler = nullptr; ObFinishPrepareMigrationDag *finish_prepare_migration_dag = nullptr; + ObDagPrio::ObDagPrioEnum prio = ObDagPrio::DAG_PRIO_MAX; if (!is_inited_) { ret = OB_NOT_INIT; @@ -1533,8 +1547,10 @@ int ObFinishPrepareMigrationTask::generate_prepare_initial_dag_() } else if (OB_ISNULL(finish_prepare_migration_dag = static_cast(this->get_dag()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("finish prepare migration dag should not be NULL", K(ret), KP(finish_prepare_migration_dag)); + } else if (OB_FAIL(ObMigrationUtils::get_dag_priority(ctx_->arg_.type_, prio))) { + LOG_WARN("failed to get dag priority", K(ret)); } else { - if (OB_FAIL(scheduler->alloc_dag(initial_prepare_dag))) { + if (OB_FAIL(scheduler->alloc_dag_with_priority(prio, initial_prepare_dag))) { LOG_WARN("failed to alloc initial prepare migration dag ", K(ret)); } else if (OB_FAIL(initial_prepare_dag->init(dag_net_))) { LOG_WARN("failed to init initial migration dag", K(ret)); diff --git a/src/storage/high_availability/ob_storage_ha_service.cpp b/src/storage/high_availability/ob_storage_ha_service.cpp index d1dcdeb053..fd2a0148f4 100644 --- a/src/storage/high_availability/ob_storage_ha_service.cpp +++ b/src/storage/high_availability/ob_storage_ha_service.cpp @@ -20,6 +20,7 @@ namespace oceanbase namespace storage { +ERRSIM_POINT_DEF(EN_STORAGE_HA_SERVICE_SET_LS_MIGRATION_STATUS_HOLD); ObStorageHAService::ObStorageHAService() : is_inited_(false), thread_cond_(), @@ -134,6 +135,12 @@ void ObStorageHAService::run1() LOG_WARN("failed to do scheduler ls ha handler", K(ret)); } +#ifdef ERRSIM + if (FAILEDx(errsim_set_ls_migration_status_hold_())) { + LOG_WARN("failed to errsim set ls migration status hold", K(ret)); + } +#endif + ObThreadCondGuard guard(thread_cond_); if (has_set_stop() || wakeup_cnt_ > 0) { wakeup_cnt_ = 0; @@ -230,6 +237,53 @@ int ObStorageHAService::do_ha_handler_(const share::ObLSID &ls_id) return ret; } +#ifdef ERRSIM +int ObStorageHAService::errsim_set_ls_migration_status_hold_() +{ + int ret = OB_SUCCESS; + ObLSHandle ls_handle; + ObLS *ls = nullptr; + const ObMigrationStatus migration_status = ObMigrationStatus::OB_MIGRATION_STATUS_HOLD; + const bool write_slog = true; + + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("storage ha service do not init", K(ret)); + } else { + ret = EN_STORAGE_HA_SERVICE_SET_LS_MIGRATION_STATUS_HOLD ? : OB_SUCCESS; + const ObAddr &self = GCONF.self_addr_; + if (OB_FAIL(ret)) { + STORAGE_LOG(ERROR, "fake EN_STORAGE_HA_SERVICE_SET_LS_MIGRATION_STATUS_HOLD", K(ret)); + //overwrite ret + ret = OB_SUCCESS; + const ObString &errsim_server = GCONF.errsim_migration_src_server_addr.str(); + if (!errsim_server.empty()) { + common::ObAddr tmp_errsim_addr; + if (OB_FAIL(tmp_errsim_addr.parse_from_string(errsim_server))) { + LOG_WARN("failed to parse from string", K(ret), K(errsim_server)); + } else if (self != tmp_errsim_addr) { + //do nothing + } else { + const int64_t errsim_migration_ls_id = GCONF.errsim_migration_ls_id; + const ObLSID ls_id(errsim_migration_ls_id); + if (errsim_migration_ls_id <= 0 || !ls_id.is_valid()) { + //do nothing + } else if (OB_FAIL(ls_service_->get_ls(ls_id, ls_handle, ObLSGetMod::HA_MOD))) { + LOG_WARN("failed to get ls", K(ret), K(ls_id)); + } else if (OB_ISNULL(ls = ls_handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls should not be NULL", K(ret), KP(ls), K(ls_id)); + } else if (OB_FAIL(ls->set_migration_status(migration_status, ls->get_rebuild_seq(), write_slog))) { + LOG_WARN("failed to set migration status", K(ret), KPC(ls), K(ls_id)); + } + } + } + + } + } + return ret; +} +#endif } } diff --git a/src/storage/high_availability/ob_storage_ha_service.h b/src/storage/high_availability/ob_storage_ha_service.h index 069df1dbc9..7b596c25d0 100644 --- a/src/storage/high_availability/ob_storage_ha_service.h +++ b/src/storage/high_availability/ob_storage_ha_service.h @@ -44,6 +44,10 @@ private: int scheduler_ls_ha_handler_(); int do_ha_handler_(const share::ObLSID &ls_id); +#ifdef ERRSIM + int errsim_set_ls_migration_status_hold_(); +#endif + private: // TODO(zeyong): change SCHEDULER_WAIT_TIME_MS to 5 min when rs use rpc to wake up the ha service in 4.3 diff --git a/src/storage/high_availability/ob_storage_ha_struct.cpp b/src/storage/high_availability/ob_storage_ha_struct.cpp index 00fbcb279d..7835324689 100644 --- a/src/storage/high_availability/ob_storage_ha_struct.cpp +++ b/src/storage/high_availability/ob_storage_ha_struct.cpp @@ -27,6 +27,7 @@ namespace oceanbase namespace storage { ERRSIM_POINT_DEF(EN_REBUILD_FAILED_STATUS); +ERRSIM_POINT_DEF(ALLOW_MIGRATION_STATUS_CHANGED); /******************ObMigrationOpType*********************/ static const char *migration_op_type_strs[] = { @@ -384,7 +385,8 @@ int ObMigrationStatusHelper::check_transfer_dest_ls_status_for_ls_gc( } else if (ObMigrationStatus::OB_MIGRATION_STATUS_NONE != dest_ls_status && ObMigrationStatus::OB_MIGRATION_STATUS_MIGRATE_WAIT != dest_ls_status && ObMigrationStatus::OB_MIGRATION_STATUS_ADD_WAIT != dest_ls_status - && ObMigrationStatus::OB_MIGRATION_STATUS_REBUILD_WAIT != dest_ls_status) { + && ObMigrationStatus::OB_MIGRATION_STATUS_REBUILD_WAIT != dest_ls_status + && ObMigrationStatus::OB_MIGRATION_STATUS_HOLD != dest_ls_status) { allow_gc = true; LOG_INFO("transfer dest ls check transfer status passed", K(ret), K(transfer_ls_id), K(dest_ls_status)); } else if (OB_FAIL(check_transfer_dest_tablet_for_ls_gc(dest_ls, tablet_id, transfer_scn, need_wait_dest_ls_replay, allow_gc))) { @@ -774,6 +776,17 @@ int ObMigrationStatusHelper::check_can_change_status( } } } + +#ifdef ERRSIM + if (OB_SUCC(ret)) { + ret = ALLOW_MIGRATION_STATUS_CHANGED ? : OB_SUCCESS; + if (OB_FAIL(ret)) { + can_change = true; + ret = OB_SUCCESS; + } + } +#endif + return ret; } @@ -1096,6 +1109,38 @@ int ObMigrationUtils::get_ls_rebuild_seq(const uint64_t tenant_id, return ret; } +int ObMigrationUtils::get_dag_priority( + const ObMigrationOpType::TYPE &type, + ObDagPrio::ObDagPrioEnum &prio) +{ + int ret = OB_SUCCESS; + if (!ObMigrationOpType::is_valid(type)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid argument", K(ret), K(type)); + } else { + switch (type) { + case ObMigrationOpType::TYPE::ADD_LS_OP: { + prio = ObDagPrio::DAG_PRIO_HA_HIGH; + break; + } + case ObMigrationOpType::TYPE::MIGRATE_LS_OP: { + prio = ObDagPrio::DAG_PRIO_HA_MID; + break; + } + case ObMigrationOpType::TYPE::REBUILD_LS_OP: { + prio = ObDagPrio::DAG_PRIO_HA_HIGH; + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("migration op type not expected", K(ret), K(type)); + break; + } + } + } + return ret; +} + /******************ObCopyTableKeyInfo*********************/ ObCopyTableKeyInfo::ObCopyTableKeyInfo() : src_table_key_(), diff --git a/src/storage/high_availability/ob_storage_ha_struct.h b/src/storage/high_availability/ob_storage_ha_struct.h index dc81d9c8e7..b9aeb850ff 100644 --- a/src/storage/high_availability/ob_storage_ha_struct.h +++ b/src/storage/high_availability/ob_storage_ha_struct.h @@ -23,6 +23,7 @@ #include "storage/blocksstable/ob_datum_rowkey.h" #include "storage/blocksstable/ob_logic_macro_id.h" #include "share/ls/ob_ls_i_life_manager.h" +#include "share/scheduler/ob_dag_scheduler_config.h" namespace oceanbase @@ -274,6 +275,9 @@ struct ObMigrationUtils const uint64_t tenant_id, const share::ObLSID &ls_id, int64_t &rebuild_seq); + static int get_dag_priority( + const ObMigrationOpType::TYPE &type, + share::ObDagPrio::ObDagPrioEnum &prio); }; struct ObCopyTableKeyInfo final diff --git a/src/storage/high_availability/ob_tablet_backfill_tx.cpp b/src/storage/high_availability/ob_tablet_backfill_tx.cpp index 52cf57fe88..5d9c3f36e0 100644 --- a/src/storage/high_availability/ob_tablet_backfill_tx.cpp +++ b/src/storage/high_availability/ob_tablet_backfill_tx.cpp @@ -517,6 +517,7 @@ int ObTabletBackfillTXTask::get_all_backfill_tx_tables_( table_array.reset(); ObArray minor_sstables; ObArray memtables; + const int64_t emergency_sstable_count = ObTabletTableStore::EMERGENCY_SSTABLE_CNT; if (!is_inited_) { ret = OB_NOT_INIT; @@ -535,7 +536,11 @@ int ObTabletBackfillTXTask::get_all_backfill_tx_tables_( } } else { // The backfill of sstable needs to start with a larger start_scn - if (OB_FAIL(ObTableStoreUtil::reverse_sort_minor_table_handles(minor_sstables))) { + if (minor_sstables.count() > emergency_sstable_count) { + ret = OB_TOO_MANY_SSTABLE; + LOG_WARN("transfer src tablet has too many sstable, cannot backfill, need retry", K(ret), + "table_count", minor_sstables.count(), "emergency sstable count", emergency_sstable_count); + } else if (OB_FAIL(ObTableStoreUtil::reverse_sort_minor_table_handles(minor_sstables))) { LOG_WARN("failed to sort minor tables", K(ret)); } else if (OB_FAIL(append(table_array, minor_sstables))) { LOG_WARN("failed to append minor sstables", K(ret), KPC(tablet), K(minor_sstables)); diff --git a/src/storage/high_availability/ob_transfer_handler.cpp b/src/storage/high_availability/ob_transfer_handler.cpp index 4eb0b4574b..a164106f6e 100644 --- a/src/storage/high_availability/ob_transfer_handler.cpp +++ b/src/storage/high_availability/ob_transfer_handler.cpp @@ -444,9 +444,7 @@ int ObTransferHandler::do_with_start_status_(const share::ObTransferTaskInfo &ta int tmp_ret = OB_SUCCESS; const int64_t start_ts = ObTimeUtil::current_time(); omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID())); - // TODO lana compatible - bool new_transfer = true; - LOG_INFO("[TRANSFER] start do with start status", K(task_info), K(new_transfer)); + LOG_INFO("[TRANSFER] start do with start status", K(task_info)); ObTimeoutCtx timeout_ctx; ObMySQLTransaction trans; @@ -461,6 +459,7 @@ int ObTransferHandler::do_with_start_status_(const share::ObTransferTaskInfo &ta process_perf_diagnose_info_(ObStorageHACostItemName::TRANSFER_START_BEGIN, ObStorageHADiagTaskType::TRANSFER_START, start_ts, round_, false/*is_report*/); bool commit_succ = false; + bool new_transfer = true; if (!is_inited_) { ret = OB_NOT_INIT; @@ -1053,7 +1052,8 @@ int ObTransferHandler::do_trans_transfer_start_( } else if (!task_info.is_valid() || !config_version.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("do trans transfer start get invalid argument", K(ret), K(task_info), K(config_version)); - } else if (OB_FAIL(do_tx_start_transfer_out_(task_info, trans, transaction::ObTxDataSourceType::START_TRANSFER_OUT))) { + } else if (OB_FAIL(do_tx_start_transfer_out_(task_info, trans, + transaction::ObTxDataSourceType::START_TRANSFER_OUT, SCN::min_scn(), nullptr))) { LOG_WARN("failed to do tx start transfer out", K(ret), K(task_info)); } else if (OB_FAIL(check_config_version_(config_version))) { LOG_WARN("failed to check config version", K(ret), K(task_info)); @@ -1192,10 +1192,13 @@ int ObTransferHandler::do_trans_transfer_start_v2_( src_info.cluster_id_ = GCONF.cluster_id; omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID())); SCN data_end_scn; + ObArray tablet_list; + ObArray move_tx_ids; int64_t move_tx_count = 0; int64_t start_time = ObTimeUtil::current_time(); int64_t transfer_out_prepare_cost = 0; int64_t wait_tablet_write_end_cost = 0; + int64_t filter_tx_cost = 0; int64_t transfer_out_cost = 0; int64_t wait_src_replay_cost = 0; int64_t get_transfer_out_scn_cost = 0; @@ -1218,9 +1221,17 @@ int ObTransferHandler::do_trans_transfer_start_v2_( } else if (OB_FAIL(get_ls_leader_(task_info.src_ls_id_, src_ls_leader))) { LOG_WARN("failed to get src ls leader", K(ret), K(task_info)); } else if (FALSE_IT(src_info.src_addr_ = src_ls_leader)) { + } else { + for (int64_t idx = 0; OB_SUCC(ret) && idx < task_info.tablet_list_.count(); idx++) { + if (OB_FAIL(tablet_list.push_back(task_info.tablet_list_.at(idx).tablet_id()))) { + LOG_WARN("push to array failed", KR(ret)); + } + } + } + if (OB_FAIL(ret)) { // MDS transaction operation for block tablet write } else if (OB_FAIL(do_tx_start_transfer_out_(task_info, trans, - transaction::ObTxDataSourceType::START_TRANSFER_OUT_PREPARE))) { + transaction::ObTxDataSourceType::START_TRANSFER_OUT_PREPARE, SCN::min_scn(), nullptr))) { LOG_WARN("failed to do tx start transfer prepare", K(ret), K(task_info)); } else if (STEP_COST_AND_CHECK_TIMEOUT(transfer_out_prepare_cost)) { // resubmit tx log promise transfer tablet redo complete @@ -1230,7 +1241,11 @@ int ObTransferHandler::do_trans_transfer_start_v2_( } else if (!data_end_scn.is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("transfer data_end_scn is invalid", K(ret), K(task_info), K(data_end_scn)); - } else if (OB_FAIL(do_tx_start_transfer_out_(task_info, trans, transaction::ObTxDataSourceType::START_TRANSFER_OUT_V2, data_end_scn))) { + } else if (OB_FAIL(ls_->filter_tx_need_transfer(tablet_list, data_end_scn, move_tx_ids))) { + LOG_WARN("filter tx need transfer", KR(ret), K(task_info)); + } else if (STEP_COST_AND_CHECK_TIMEOUT(filter_tx_cost)) { + } else if (OB_FAIL(do_tx_start_transfer_out_(task_info, trans, + transaction::ObTxDataSourceType::START_TRANSFER_OUT_V2, data_end_scn, &move_tx_ids))) { LOG_WARN("failed to do tx start transfer out", K(ret), K(task_info)); } else if (STEP_COST_AND_CHECK_TIMEOUT(transfer_out_cost)) { } else if (OB_FAIL(get_start_transfer_out_scn_(task_info, timeout_ctx, start_scn))) { @@ -1244,7 +1259,8 @@ int ObTransferHandler::do_trans_transfer_start_v2_( LOG_WARN("failed to get transfer tablets meta", K(ret), K(task_info)); } else if (STEP_COST_AND_CHECK_TIMEOUT(get_tablets_meta_cost)) { // move tx - } else if (OB_FAIL(do_move_tx_to_dest_ls_(task_info, timeout_ctx, trans, data_end_scn, start_scn, move_tx_count))) { + } else if (move_tx_ids.count() > 0 && OB_FAIL(do_move_tx_to_dest_ls_(task_info, timeout_ctx, trans, + data_end_scn, start_scn, tablet_list, move_tx_ids, move_tx_count))) { LOG_WARN("failed to do move tx to dest_ls", K(ret), K(task_info)); } else if (STEP_COST_AND_CHECK_TIMEOUT(move_tx_cost)) { // transfer in @@ -1255,16 +1271,18 @@ int ObTransferHandler::do_trans_transfer_start_v2_( LOG_WARN("failed to update transfer status", K(ret), K(task_info)); } - LOG_INFO("[TRANSFER] finish do trans transfer start", K(ret), K(task_info), "cost", ObTimeUtil::current_time() - start_time, - K(transfer_out_prepare_cost), - K(wait_tablet_write_end_cost), - K(transfer_out_cost), - K(get_transfer_out_scn_cost), - K(wait_src_replay_cost), - K(get_tablets_meta_cost), - K(move_tx_cost), - K(transfer_in_cost), - K(move_tx_count)); + LOG_INFO("[TRANSFER] finish transfer start", K(ret), K(task_info), "cost", ObTimeUtil::current_time() - start_time, + K(transfer_out_prepare_cost), + K(wait_tablet_write_end_cost), + K(filter_tx_cost), + K(transfer_out_cost), + K(get_transfer_out_scn_cost), + K(wait_src_replay_cost), + K(get_tablets_meta_cost), + K(move_tx_cost), + K(transfer_in_cost), + K(move_tx_count), + K(move_tx_ids)); return ret; } @@ -1359,7 +1377,8 @@ int ObTransferHandler::do_tx_start_transfer_out_( const share::ObTransferTaskInfo &task_info, common::ObMySQLTransaction &trans, const transaction::ObTxDataSourceType data_source_type, - SCN data_end_scn) + SCN data_end_scn, + ObIArray *move_tx_ids) { LOG_INFO("[TRANSFER] register start transfer out", K(task_info), K(data_source_type)); int ret = OB_SUCCESS; @@ -1368,7 +1387,7 @@ int ObTransferHandler::do_tx_start_transfer_out_( ObArenaAllocator allocator; SCN dest_base_scn; const int64_t start_ts = ObTimeUtil::current_time(); - + const int64_t ENABLE_FILTER_TX_LIST_LIMIT = 1000; if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("transfer handler do not init", K(ret)); @@ -1388,9 +1407,17 @@ int ObTransferHandler::do_tx_start_transfer_out_( start_transfer_out_info.transfer_epoch_ = task_info.task_id_.id(); start_transfer_out_info.task_id_ = task_info.task_id_; start_transfer_out_info.data_version_ = DEFAULT_MIN_DATA_VERSION; + start_transfer_out_info.filter_tx_need_transfer_ = false; if (OB_FAIL(start_transfer_out_info.tablet_list_.assign(task_info.tablet_list_))) { LOG_WARN("failed to assign transfer tablet list", K(ret), K(task_info)); - } else { + } else if (OB_NOT_NULL(move_tx_ids) && move_tx_ids->count() <= ENABLE_FILTER_TX_LIST_LIMIT) { + // if has too many tx_ids, we just use data_end_scn to filter + start_transfer_out_info.filter_tx_need_transfer_ = true; + if (OB_FAIL(start_transfer_out_info.move_tx_ids_.assign(*move_tx_ids))) { + LOG_WARN("assign failed", KR(ret), K(move_tx_ids)); + } + } + if (OB_SUCC(ret)) { int64_t buf_len = start_transfer_out_info.get_serialize_size(); int64_t pos = 0; char *buf = (char*)allocator.alloc(buf_len); @@ -2645,6 +2672,8 @@ int ObTransferHandler::do_move_tx_to_dest_ls_(const share::ObTransferTaskInfo &t ObMySQLTransaction &trans, const SCN data_end_scn, const SCN transfer_scn, + ObIArray &tablet_list, + ObIArray &move_tx_ids, int64_t &move_tx_count) { LOG_INFO("[TRANSFER] do_move_tx_to_dest_ls_", K(task_info), K(data_end_scn)); @@ -2661,19 +2690,12 @@ int ObTransferHandler::do_move_tx_to_dest_ls_(const share::ObTransferTaskInfo &t int64_t tx_count = 0; int64_t buf_len = 0; int64_t collect_count = 0; - ObArray tablet_list; - for (int64_t idx = 0; OB_SUCC(ret) && idx < task_info.tablet_list_.count(); idx++) { - if (OB_FAIL(tablet_list.push_back(task_info.tablet_list_.at(idx).tablet_id()))) { - LOG_WARN("push to array failed", KR(ret)); - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(MTL(ObLSService*)->get_ls(task_info.src_ls_id_,src_ls_handle, ObLSGetMod::STORAGE_MOD))) { + if (OB_FAIL(MTL(ObLSService*)->get_ls(task_info.src_ls_id_,src_ls_handle, ObLSGetMod::STORAGE_MOD))) { LOG_WARN("get ls failed", KR(ret), K(task_info)); } else if (OB_FAIL(src_ls_handle.get_ls()->collect_tx_ctx(task_info.dest_ls_id_, data_end_scn, - const_cast&>(tablet_list), - tx_count, + tablet_list, + move_tx_ids, collect_count, collect_res.args_))) { LOG_WARN("collect tx ctx failed", KR(ret), K(task_info)); @@ -2723,7 +2745,7 @@ int ObTransferHandler::do_move_tx_to_dest_ls_(const share::ObTransferTaskInfo &t int64_t end_time = ObTimeUtility::current_time(); LOG_INFO("do_move_tx_to_dest_ls_", KR(ret), "cost", end_time-start_time, K(task_info), - "tx_count", collect_res.args_.count(), + "move_tx_count", collect_res.args_.count(), "buf_size", buf_len); return ret; } diff --git a/src/storage/high_availability/ob_transfer_handler.h b/src/storage/high_availability/ob_transfer_handler.h index 553eb34a28..d61f22d0d2 100644 --- a/src/storage/high_availability/ob_transfer_handler.h +++ b/src/storage/high_availability/ob_transfer_handler.h @@ -193,6 +193,8 @@ private: ObMySQLTransaction &trans, const SCN data_end_scn, const SCN transfer_scn, + ObIArray &tablet_list, + ObIArray &move_tx_ids, int64_t &move_tx_count); int start_trans_( ObTimeoutCtx &timeout_ctx, @@ -205,7 +207,8 @@ private: const share::ObTransferTaskInfo &task_info, common::ObMySQLTransaction &trans, const transaction::ObTxDataSourceType data_source_type, - SCN data_end_scn = SCN::min_scn()); + SCN data_end_scn, + ObIArray *move_tx_ids); int lock_transfer_task_( const share::ObTransferTaskInfo &task_info, common::ObISQLClient &trans); diff --git a/src/storage/high_availability/ob_transfer_struct.cpp b/src/storage/high_availability/ob_transfer_struct.cpp index a967601add..b956c20b8f 100644 --- a/src/storage/high_availability/ob_transfer_struct.cpp +++ b/src/storage/high_availability/ob_transfer_struct.cpp @@ -34,7 +34,9 @@ ObTXStartTransferOutInfo::ObTXStartTransferOutInfo() task_id_(), data_end_scn_(), transfer_epoch_(0), - data_version_(DEFAULT_MIN_DATA_VERSION) + data_version_(DEFAULT_MIN_DATA_VERSION), + filter_tx_need_transfer_(false), + move_tx_ids_() { } @@ -47,6 +49,8 @@ void ObTXStartTransferOutInfo::reset() data_end_scn_.reset(); transfer_epoch_ = 0; data_version_ = 0; + filter_tx_need_transfer_ = false; + move_tx_ids_.reset(); } bool ObTXStartTransferOutInfo::is_valid() const @@ -65,6 +69,8 @@ int ObTXStartTransferOutInfo::assign(const ObTXStartTransferOutInfo &start_trans LOG_WARN("assign start transfer out info get invalid argument", K(ret), K(start_transfer_out_info)); } else if (OB_FAIL(tablet_list_.assign(start_transfer_out_info.tablet_list_))) { LOG_WARN("failed to assign start transfer out info", K(ret), K(start_transfer_out_info)); + } else if (OB_FAIL(move_tx_ids_.assign(start_transfer_out_info.move_tx_ids_))) { + LOG_WARN("failed to assign move_tx_ids", K(ret), K(start_transfer_out_info)); } else { src_ls_id_ = start_transfer_out_info.src_ls_id_; dest_ls_id_ = start_transfer_out_info.dest_ls_id_; @@ -72,11 +78,13 @@ int ObTXStartTransferOutInfo::assign(const ObTXStartTransferOutInfo &start_trans data_end_scn_ = start_transfer_out_info.data_end_scn_; transfer_epoch_ = start_transfer_out_info.transfer_epoch_; data_version_ = start_transfer_out_info.data_version_; + filter_tx_need_transfer_ = start_transfer_out_info.filter_tx_need_transfer_; } return ret; } -OB_SERIALIZE_MEMBER(ObTXStartTransferOutInfo, src_ls_id_, dest_ls_id_, tablet_list_, task_id_, data_end_scn_, transfer_epoch_, data_version_); +OB_SERIALIZE_MEMBER(ObTXStartTransferOutInfo, src_ls_id_, dest_ls_id_, tablet_list_, task_id_, + data_end_scn_, transfer_epoch_, data_version_, filter_tx_need_transfer_, move_tx_ids_); ObTXStartTransferInInfo::ObTXStartTransferInInfo() : src_ls_id_(), @@ -1010,4 +1018,4 @@ int ObTransferRelatedInfo::get_related_info_task_id(share::ObTransferTaskID &tas task_id = get_task_id_(); } return ret; -} \ No newline at end of file +} diff --git a/src/storage/high_availability/ob_transfer_struct.h b/src/storage/high_availability/ob_transfer_struct.h index c9a5f80142..65eb9a04eb 100644 --- a/src/storage/high_availability/ob_transfer_struct.h +++ b/src/storage/high_availability/ob_transfer_struct.h @@ -37,8 +37,10 @@ public: void reset(); bool is_valid() const; int assign(const ObTXStartTransferOutInfo &start_transfer_out_info); + bool empty_tx() { return filter_tx_need_transfer_ && move_tx_ids_.count() == 0; } - TO_STRING_KV(K_(src_ls_id), K_(dest_ls_id), K_(tablet_list), K_(task_id), K_(data_end_scn), K_(transfer_epoch), K_(data_version)); + TO_STRING_KV(K_(src_ls_id), K_(dest_ls_id), K_(tablet_list), K_(task_id), K_(data_end_scn), K_(transfer_epoch), K_(data_version), + K_(filter_tx_need_transfer), K_(move_tx_ids)); share::ObLSID src_ls_id_; share::ObLSID dest_ls_id_; @@ -47,6 +49,8 @@ public: share::SCN data_end_scn_; int64_t transfer_epoch_; uint64_t data_version_; //transfer_dml_ctrl_42x # placeholder + bool filter_tx_need_transfer_; + common::ObSEArray move_tx_ids_; DISALLOW_COPY_AND_ASSIGN(ObTXStartTransferOutInfo); }; diff --git a/src/storage/ls/ob_ls.cpp b/src/storage/ls/ob_ls.cpp old mode 100644 new mode 100755 index 4053e259a1..5970deef66 --- a/src/storage/ls/ob_ls.cpp +++ b/src/storage/ls/ob_ls.cpp @@ -2418,6 +2418,7 @@ int ObLS::set_migration_status( int ret = OB_SUCCESS; share::ObLSRestoreStatus restore_status; WRLockGuard guard(meta_rwlock_); + bool allow_read = false; if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -2438,8 +2439,9 @@ int ObLS::set_migration_status( LOG_WARN("failed to get restore status", K(ret), K(ls_meta_)); } else if (OB_FAIL(ls_meta_.set_migration_status(migration_status, write_slog))) { LOG_WARN("failed to set migration status", K(ret), K(migration_status)); - } else if (ObMigrationStatus::OB_MIGRATION_STATUS_NONE == migration_status - && restore_status.is_none()) { + } else if (OB_FAIL(inner_check_allow_read_(migration_status, restore_status, allow_read))) { + LOG_WARN("failed to check allow to read", K(ret), K(migration_status), K(restore_status)); + } else if (allow_read) { ls_tablet_svr_.enable_to_read(); } else { ls_tablet_svr_.disable_to_read(); @@ -2454,6 +2456,7 @@ int ObLS::set_restore_status( int ret = OB_SUCCESS; ObMigrationStatus migration_status = ObMigrationStatus::OB_MIGRATION_STATUS_MAX; WRLockGuard guard(meta_rwlock_); + bool allow_read = false; if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -2474,8 +2477,9 @@ int ObLS::set_restore_status( LOG_WARN("failed to get migration status", K(ret), K(ls_meta_)); } else if (OB_FAIL(ls_meta_.set_restore_status(restore_status))) { LOG_WARN("failed to set restore status", K(ret), K(restore_status)); - } else if (ObMigrationStatus::OB_MIGRATION_STATUS_NONE == migration_status - && restore_status.is_none()) { + } else if (OB_FAIL(inner_check_allow_read_(migration_status, restore_status, allow_read))) { + LOG_WARN("failed to check allow to read", K(ret), K(migration_status), K(restore_status)); + } else if (allow_read) { ls_tablet_svr_.enable_to_read(); } else { ls_tablet_svr_.disable_to_read(); @@ -2591,4 +2595,40 @@ void ObLS::clear_delay_resource_recycle() LOG_INFO("clear delay resource recycle", KPC(this)); } } + +int ObLS::check_allow_read(bool &allow_to_read) +{ + int ret = OB_SUCCESS; + share::ObLSRestoreStatus restore_status; + ObMigrationStatus migration_status = ObMigrationStatus::OB_MIGRATION_STATUS_MAX; + allow_to_read = false; + //allow ls is not init because create ls will schedule this interface + if (OB_FAIL(ls_meta_.get_migration_and_restore_status(migration_status, restore_status))) { + LOG_WARN("failed to get migration and restore status"); + } else if (OB_FAIL(inner_check_allow_read_(migration_status, restore_status, allow_to_read))) { + LOG_WARN("failed to do inner check allow read", K(ret), K(migration_status), K(restore_status)); + } + return ret; +} + +int ObLS::inner_check_allow_read_( + const ObMigrationStatus &migration_status, + const share::ObLSRestoreStatus &restore_status, + bool &allow_read) +{ + int ret = OB_SUCCESS; + allow_read = false; + if (!ObMigrationStatusHelper::is_valid(migration_status) || !restore_status.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("inner check allow read get invalid argument", K(ret), K(migration_status), K(restore_status)); + } else if ((ObMigrationStatus::OB_MIGRATION_STATUS_NONE == migration_status + || ObMigrationStatus::OB_MIGRATION_STATUS_HOLD == migration_status) + && restore_status.is_none()) { + allow_read = true; + } else { + allow_read = false; + } + return ret; +} + } diff --git a/src/storage/ls/ob_ls.h b/src/storage/ls/ob_ls.h index 3720da82a7..c941df2deb 100644 --- a/src/storage/ls/ob_ls.h +++ b/src/storage/ls/ob_ls.h @@ -386,6 +386,7 @@ public: int try_sync_reserved_snapshot(const int64_t new_reserved_snapshot, const bool update_flag); int check_can_replay_clog(bool &can_replay); int check_ls_need_online(bool &need_online); + int check_allow_read(bool &allow_read); // for delaying the resource recycle after correctness issue bool need_delay_resource_recycle() const; @@ -833,10 +834,12 @@ public: DELEGATE_WITH_RET(ls_tx_svr_, get_tx_ctx_count, int); DELEGATE_WITH_RET(ls_tx_svr_, get_active_tx_count, int); DELEGATE_WITH_RET(ls_tx_svr_, print_all_tx_ctx, int); + DELEGATE_WITH_RET(ls_tx_svr_, retry_apply_start_working_log, int); //dup table ls meta interface CONST_DELEGATE_WITH_RET(dup_table_ls_handler_, get_dup_table_ls_meta, int); DELEGATE_WITH_RET(dup_table_ls_handler_, set_dup_table_ls_meta, int); + DELEGATE_WITH_RET(ls_tx_svr_, filter_tx_need_transfer, int); // for transfer to modify active tx ctx state DELEGATE_WITH_RET(ls_tx_svr_, transfer_out_tx_op, int); @@ -938,6 +941,10 @@ public: DELEGATE_WITH_RET(reserved_snapshot_mgr_, add_dependent_medium_tablet, int); DELEGATE_WITH_RET(reserved_snapshot_mgr_, del_dependent_medium_tablet, int); int set_ls_migration_gc(bool &allow_gc); + int inner_check_allow_read_( + const ObMigrationStatus &migration_status, + const share::ObLSRestoreStatus &restore_status, + bool &allow_read); private: // StorageBaseUtil diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index b3f4d97dda..96b9ca96a9 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -5892,6 +5892,11 @@ int ObLSTabletService::get_ls_min_end_scn( } } } + // now tx_data contains mds tx_op to remove retain_ctx + // so we need wait ls_checkpoint advance to recycle tx_data + if (ls_checkpoint < min_end_scn_from_latest_tablets) { + min_end_scn_from_latest_tablets = ls_checkpoint; + } LOG_INFO("get ls min end scn finish", K(ls_checkpoint)); } return ret; @@ -6297,20 +6302,17 @@ int ObLSTabletService::build_tablet_iter(ObLSTabletFastIter &iter, const bool ex int ObLSTabletService::set_allow_to_read_(ObLS *ls) { int ret = OB_SUCCESS; - ObMigrationStatus migration_status = ObMigrationStatus::OB_MIGRATION_STATUS_MAX; - share::ObLSRestoreStatus restore_status; + bool allow_read = false; if (OB_ISNULL(ls)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("set allow to read get invalid argument", K(ret), KP(ls)); } else { - if (OB_FAIL(ls->get_migration_and_restore_status(migration_status, restore_status))) { - LOG_WARN("failed to get ls migration and restore status", K(ret), KPC(ls)); - } else if ((ObMigrationStatus::OB_MIGRATION_STATUS_NONE != migration_status - && ObMigrationStatus::OB_MIGRATION_STATUS_HOLD != migration_status) - || ObLSRestoreStatus::NONE != restore_status) { + if (OB_FAIL(ls->check_allow_read(allow_read))) { + LOG_WARN("failed to check allow read", K(ret), KPC(ls)); + } else if (!allow_read) { allow_to_read_mgr_.disable_to_read(); - FLOG_INFO("set ls do not allow to read", KPC(ls), K(migration_status), K(restore_status)); + FLOG_INFO("set ls do not allow to read", KPC(ls)); } else { allow_to_read_mgr_.enable_to_read(); } diff --git a/src/storage/ls/ob_ls_tx_service.cpp b/src/storage/ls/ob_ls_tx_service.cpp index e5f5f4eb87..9484ae5525 100644 --- a/src/storage/ls/ob_ls_tx_service.cpp +++ b/src/storage/ls/ob_ls_tx_service.cpp @@ -899,6 +899,18 @@ int ObLSTxService::print_all_tx_ctx(const int64_t print_num) return ret; } +int ObLSTxService::retry_apply_start_working_log() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(mgr_)) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "not init", KR(ret), K_(ls_id)); + } else { + ret = mgr_->retry_apply_start_working_log(); + } + return ret; +} + int ObLSTxService::set_max_replay_commit_version(share::SCN commit_version) { int ret = OB_SUCCESS; @@ -924,25 +936,28 @@ int ObLSTxService::check_tx_blocked(bool &tx_blocked) const } return ret; } +int ObLSTxService::filter_tx_need_transfer(ObIArray &tablet_list, + const share::SCN data_end_scn, + ObIArray &move_tx_ids) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(mgr_->filter_tx_need_transfer(tablet_list, data_end_scn, move_tx_ids))) { + TRANS_LOG(WARN, "for each tx ctx error", KR(ret)); + } + return ret; +} -int ObLSTxService::transfer_out_tx_op(int64_t except_tx_id, - const share::SCN data_end_scn, - const share::SCN op_scn, - transaction::NotifyType op_type, - bool is_replay, - share::ObLSID dest_ls_id, - int64_t transfer_epoch, +int ObLSTxService::transfer_out_tx_op(const ObTransferOutTxParam ¶m, int64_t &active_tx_count, int64_t &op_tx_count) { int ret = OB_SUCCESS; int64_t start_time = ObTimeUtility::current_time(); - if (OB_FAIL(mgr_->transfer_out_tx_op(except_tx_id, data_end_scn, op_scn, op_type, is_replay, - dest_ls_id, transfer_epoch, active_tx_count, op_tx_count))) { + if (OB_FAIL(mgr_->transfer_out_tx_op(param, active_tx_count, op_tx_count))) { TRANS_LOG(WARN, "for each tx ctx error", KR(ret)); } int64_t end_time = ObTimeUtility::current_time(); - LOG_INFO("transfer_out_tx_op", KR(ret), K(op_type), "cost", end_time - start_time, K(active_tx_count), K(op_tx_count)); + LOG_INFO("transfer_out_tx_op", KR(ret), "cost", end_time - start_time, K(active_tx_count), K(op_tx_count)); return ret; } @@ -961,18 +976,17 @@ int ObLSTxService::wait_tx_write_end(ObTimeoutCtx &timeout_ctx) int ObLSTxService::collect_tx_ctx(const ObLSID dest_ls_id, const SCN log_scn, const ObIArray &tablet_list, - int64_t &tx_count, + const ObIArray &move_tx_ids, int64_t &collect_count, ObIArray &res) { int ret = OB_SUCCESS; int64_t start_time = ObTimeUtility::current_time(); - if (OB_FAIL(mgr_->collect_tx_ctx(dest_ls_id, log_scn, tablet_list, tx_count, collect_count, res))) { + if (OB_FAIL(mgr_->collect_tx_ctx(dest_ls_id, log_scn, tablet_list, move_tx_ids, collect_count, res))) { TRANS_LOG(WARN, "for each tx ctx error", KR(ret)); } int64_t end_time = ObTimeUtility::current_time(); - LOG_INFO("collect_tx_ctx", KR(ret), K(ls_id_), "cost_us", end_time - start_time, - K(tx_count), K(collect_count)); + LOG_INFO("collect_tx_ctx", KR(ret), K(ls_id_), "cost_us", end_time - start_time, K(collect_count)); return ret; } diff --git a/src/storage/ls/ob_ls_tx_service.h b/src/storage/ls/ob_ls_tx_service.h index 3ded9ac2d7..af1111399a 100644 --- a/src/storage/ls/ob_ls_tx_service.h +++ b/src/storage/ls/ob_ls_tx_service.h @@ -34,6 +34,7 @@ namespace storage class ObLS; struct ObTxCtxMoveArg; struct ObTransferMoveTxParam; +struct ObTransferOutTxParam; } namespace transaction @@ -158,6 +159,8 @@ public: int get_active_tx_count(int64_t &active_tx_count); int print_all_tx_ctx(const int64_t print_num); + int retry_apply_start_working_log(); + public: int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, const share::SCN &scn); @@ -174,20 +177,17 @@ public: int get_common_checkpoint_info( ObIArray &common_checkpoint_array); - int transfer_out_tx_op(int64_t except_tx_id, - const share::SCN data_end_scn, - const share::SCN op_scn, - transaction::NotifyType op_type, - bool is_replay, - share::ObLSID dest_ls_id, - int64_t transfer_epoch, + int filter_tx_need_transfer(ObIArray &tablet_list, + const share::SCN data_end_scn, + ObIArray &move_tx_ids); + int transfer_out_tx_op(const ObTransferOutTxParam ¶m, int64_t &active_tx_count, int64_t &op_tx_count); int wait_tx_write_end(ObTimeoutCtx &timeout_ctx); int collect_tx_ctx(const share::ObLSID dest_ls_id, const share::SCN log_scn, const ObIArray &tablet_list, - int64_t &tx_count, + const ObIArray &move_tx_ids, int64_t &collect_count, ObIArray &args); int move_tx_op(const ObTransferMoveTxParam &move_tx_param, diff --git a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp index e14975d50c..4f7cbb78e3 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp @@ -1323,9 +1323,7 @@ int ObTransCallbackMgr::acquire_callback_list(const bool new_epoch, const bool n ATOMIC_STORE(¶llel_stat_, PARALLEL_STMT); } } else if (tid == (stat >> 32)) { // same thread nested, no parallel - if (!ATOMIC_BCAS(¶llel_stat_, stat, stat + 1)) { - TRANS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "Unexpected status", K(this), K(tid_), K(ref_cnt_), K(tid)); - } + ATOMIC_BCAS(¶llel_stat_, stat, stat + 1); } else { // has parallel // ATOMIC_STORE(¶llel_stat_, PARALLEL_STMT); diff --git a/src/storage/memtable/ob_memtable.cpp b/src/storage/memtable/ob_memtable.cpp index 4f2b0f61d6..d7a368d49b 100644 --- a/src/storage/memtable/ob_memtable.cpp +++ b/src/storage/memtable/ob_memtable.cpp @@ -321,7 +321,7 @@ int ObMemtable::safe_to_destroy(bool &is_safe) if (!is_safe && ret == OB_STATE_NOT_MATCH) { ret = OB_SUCCESS; bool is_done = false; - share::LSN end_lsn; + LSN end_lsn; if (OB_FAIL(MTL(logservice::ObLogService*)->get_log_apply_service()-> is_apply_done(ls_handle_.get_ls()->get_ls_id(), is_done, @@ -2201,17 +2201,17 @@ int ObMemtable::estimate_phy_size(const ObStoreRowkey* start_key, const ObStoreR return ret; } -int ObMemtable::get_split_ranges(const ObStoreRowkey* start_key, const ObStoreRowkey* end_key, const int64_t part_cnt, ObIArray &range_array) +int ObMemtable::get_split_ranges(const ObStoreRange &input_range, + const int64_t part_cnt, + ObIArray &range_array) { int ret = OB_SUCCESS; + range_array.reuse(); + const ObStoreRowkey *start_key = &input_range.get_start_key(); + const ObStoreRowkey *end_key = &input_range.get_end_key(); ObMemtableKey start_mtk; ObMemtableKey end_mtk; - if (NULL == start_key) { - start_key = &ObStoreRowkey::MIN_STORE_ROWKEY; - } - if (NULL == end_key) { - end_key = &ObStoreRowkey::MAX_STORE_ROWKEY; - } + if (part_cnt < 1) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "part cnt need be greater than 1", K(ret), K(part_cnt)); @@ -2220,6 +2220,25 @@ int ObMemtable::get_split_ranges(const ObStoreRowkey* start_key, const ObStoreRo } else if (OB_FAIL(query_engine_.split_range(&start_mtk, &end_mtk, part_cnt, range_array))) { TRANS_LOG(WARN, "estimate row count fail", K(ret), K_(key)); } + + if (OB_ENTRY_NOT_EXIST == ret) { + // construct a single range if split failed + ret = OB_SUCCESS; + ObStoreRange merge_range; + merge_range.set_start_key(*start_key); + merge_range.set_end_key(*end_key); + if (OB_FAIL(range_array.push_back(merge_range))) { + STORAGE_LOG(WARN, "push back merge range to range array failed", KR(ret), K(merge_range)); + } + } + + if (OB_SUCC(ret) && !range_array.empty()) { + // set range closed or open + ObStoreRange &first_range = range_array.at(0); + ObStoreRange &last_range = range_array.at(range_array.count() - 1); + input_range.get_border_flag().inclusive_start() ? first_range.set_left_closed() : first_range.set_left_open(); + input_range.get_border_flag().inclusive_end() ? last_range.set_right_closed() : last_range.set_right_open(); + } return ret; } @@ -2269,11 +2288,14 @@ int ObMemtable::split_ranges_for_sample(const blocksstable::ObDatumRange &table_ while (!split_succ && total_split_range_count > ObMemtableRowSampleIterator::SAMPLE_MEMTABLE_RANGE_COUNT) { int tmp_ret = OB_SUCCESS; sample_memtable_ranges.reuse(); - if (OB_TMP_FAIL(try_split_range_for_sample_(table_scan_range.get_start_key().get_store_rowkey(), - table_scan_range.get_end_key().get_store_rowkey(), - total_split_range_count, - allocator, - sample_memtable_ranges))) { + ObStoreRange input_range; + input_range.set_start_key(table_scan_range.get_start_key().get_store_rowkey()); + input_range.set_end_key(table_scan_range.get_end_key().get_store_rowkey()); + input_range.is_left_open() ? input_range.set_left_open() : input_range.set_left_closed(); + input_range.is_right_open() ? input_range.set_right_open() : input_range.set_right_closed(); + + if (OB_TMP_FAIL( + try_split_range_for_sample_(input_range, total_split_range_count, allocator, sample_memtable_ranges))) { total_split_range_count = total_split_range_count / 10; TRANS_LOG(WARN, "try split range for sampling failed, shrink split range count and retry", @@ -2294,19 +2316,18 @@ int ObMemtable::split_ranges_for_sample(const blocksstable::ObDatumRange &table_ return ret; } -int64_t ObMemtable::try_split_range_for_sample_(const ObStoreRowkey &start_key, - const ObStoreRowkey &end_key, +int64_t ObMemtable::try_split_range_for_sample_(const ObStoreRange &input_range, const int64_t range_count, ObIAllocator &allocator, ObIArray &sample_memtable_ranges) { int ret = OB_SUCCESS; ObSEArray store_range_array; - if (OB_FAIL(get_split_ranges(&start_key, &end_key, range_count, store_range_array))) { + if (OB_FAIL(get_split_ranges(input_range, range_count, store_range_array))) { TRANS_LOG(WARN, "try split ranges for sample failed", KR(ret)); } else if (store_range_array.count() != range_count) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "store array count is not equal with range_count", KR(ret), K(range_count), KPC(this)); + ret = OB_ENTRY_NOT_EXIST; + TRANS_LOG(INFO, "memtable row is not enough for splitting", KR(ret), K(range_count), KPC(this)); } else { const int64_t range_count_each_chosen = range_count / (ObMemtableRowSampleIterator::SAMPLE_MEMTABLE_RANGE_COUNT - 1); @@ -2640,19 +2661,6 @@ int ObMemtable::multi_set_( } } - // 3. Roll back all rows that have been inserted if meet failure. - if (OB_FAIL(ret)) { - for (int64_t i = 0; i < mvcc_rows.count(); ++i) { - if (conflict_idx != i) { - ObMvccWriteResult &write_result = mvcc_rows[i].write_result_; - ObMvccRow* mvcc_row = mvcc_rows[i].mvcc_row_; - if (write_result.has_insert()) { - (void)mvcc_engine_.mvcc_undo(mvcc_row); - } - } - } - } - if (OB_TRANSACTION_SET_VIOLATION == ret) { ObTxIsolationLevel iso = ctx.mvcc_acc_ctx_.tx_desc_->get_isolation_level(); if (ObTxIsolationLevel::SERIAL == iso || ObTxIsolationLevel::RR == iso) { diff --git a/src/storage/memtable/ob_memtable.h b/src/storage/memtable/ob_memtable.h index b7845c9741..5a02ae1a92 100644 --- a/src/storage/memtable/ob_memtable.h +++ b/src/storage/memtable/ob_memtable.h @@ -394,7 +394,9 @@ public: void set_contain_hotspot_row() { return ATOMIC_STORE(&contain_hotspot_row_, true); } virtual int64_t get_upper_trans_version() const override; virtual int estimate_phy_size(const ObStoreRowkey* start_key, const ObStoreRowkey* end_key, int64_t& total_bytes, int64_t& total_rows) override; - virtual int get_split_ranges(const ObStoreRowkey* start_key, const ObStoreRowkey* end_key, const int64_t part_cnt, common::ObIArray &range_array) override; + virtual int get_split_ranges(const ObStoreRange &input_range, + const int64_t part_cnt, + ObIArray &range_array) override; int split_ranges_for_sample(const blocksstable::ObDatumRange &table_scan_range, const double sample_rate_percentage, ObIAllocator &allocator, @@ -573,8 +575,7 @@ private: const int64_t last_compact_cnt, const int64_t total_trans_node_count); bool ready_for_flush_(); - int64_t try_split_range_for_sample_(const ObStoreRowkey &start_key, - const ObStoreRowkey &end_key, + int64_t try_split_range_for_sample_(const ObStoreRange &input_range, const int64_t range_count, ObIAllocator &allocator, ObIArray &sample_memtable_ranges); diff --git a/src/storage/memtable/ob_memtable_interface.h b/src/storage/memtable/ob_memtable_interface.h index cfce5d0874..94b66949c9 100644 --- a/src/storage/memtable/ob_memtable_interface.h +++ b/src/storage/memtable/ob_memtable_interface.h @@ -142,22 +142,18 @@ public: total_rows = 0; return OB_SUCCESS; } - virtual int get_split_ranges(const ObStoreRowkey *start_key, - const ObStoreRowkey *end_key, + + virtual int get_split_ranges(const ObStoreRange &input_range, const int64_t part_cnt, - common::ObIArray &range_array) + ObIArray &range_array) { - UNUSEDx(start_key, end_key); + UNUSEDx(input_range); int ret = OB_SUCCESS; if (OB_UNLIKELY(part_cnt != 1)) { ret = OB_NOT_SUPPORTED; - } else { - ObStoreRange merge_range; - merge_range.set_start_key(ObStoreRowkey::MIN_STORE_ROWKEY); - merge_range.set_end_key(ObStoreRowkey::MAX_STORE_ROWKEY); - if (OB_FAIL(range_array.push_back(merge_range))) { - TRANS_LOG(ERROR, "push back to range array failed", K(ret)); - } + STORAGE_LOG(WARN, "split a single range is not supported", KR(ret), K(input_range), K(part_cnt)); + } else if (OB_FAIL(range_array.push_back(input_range))) { + STORAGE_LOG(WARN, "push back to range array failed", K(ret)); } return ret; } diff --git a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h index 2d620f426c..469f7320b5 100644 --- a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h +++ b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h @@ -122,7 +122,7 @@ class ObTenantMetaMemMgr final { public: static const int64_t THE_SIZE_OF_HEADERS = sizeof(ObFIFOAllocator::NormalPageHeader) + sizeof(ObMetaObjBufferNode); - static const int64_t NORMAL_TABLET_POOL_SIZE = (ABLOCK_SIZE - ABLOCK_HEADER_SIZE) / 2 - AOBJECT_META_SIZE - THE_SIZE_OF_HEADERS; // 3952B + static const int64_t NORMAL_TABLET_POOL_SIZE = (ABLOCK_SIZE - ABLOCK_HEADER_SIZE) / 2 - AOBJECT_META_SIZE - AOBJECT_EXTRA_INFO_SIZE - THE_SIZE_OF_HEADERS; // 3824B static const int64_t LARGE_TABLET_POOL_SIZE = 64 * 1024L - THE_SIZE_OF_HEADERS; // 65,480B static const int64_t MIN_MODE_MAX_TABLET_CNT_IN_OBJ_POOL = 10000; diff --git a/src/storage/multi_data_source/buffer_ctx.cpp b/src/storage/multi_data_source/buffer_ctx.cpp index 6b481d7865..4cf4d4ce4a 100644 --- a/src/storage/multi_data_source/buffer_ctx.cpp +++ b/src/storage/multi_data_source/buffer_ctx.cpp @@ -69,22 +69,22 @@ int BufferCtxNode::serialize(char *buf, const int64_t buf_len, int64_t &pos) con } template -int deserialize_(BufferCtx *&ctx_, int64_t type_idx, const char *buf, const int64_t buf_len, int64_t &pos) { +int deserialize_(BufferCtx *&ctx_, int64_t type_idx, const char *buf, const int64_t buf_len, int64_t &pos, ObIAllocator &allocator) { int ret = OB_SUCCESS; MDS_TG(10_ms); if (IDX == type_idx) { using ImplType = GET_CTX_TYPE_BY_TUPLE_IDX(IDX); ImplType *p_impl = nullptr; set_mds_mem_check_thread_local_info(MdsWriter(WriterType::UNKNOWN_WRITER, 0), typeid(ImplType).name()); - if (OB_ISNULL(p_impl = (ImplType *)MTL(ObTenantMdsService*)->get_buffer_ctx_allocator().alloc(sizeof(ImplType), - ObMemAttr(MTL_ID(), - "MDS_CTX_DESE", - ObCtxIds::MDS_CTX_ID)))) { + if (OB_ISNULL(p_impl = (ImplType *)allocator.alloc(sizeof(ImplType), + ObMemAttr(MTL_ID(), + "MDS_CTX_DESE", + ObCtxIds::MDS_CTX_ID)))) { ret = OB_ALLOCATE_MEMORY_FAILED; MDS_LOG(ERROR, "fail to alloc buffer ctx memory", KR(ret), K(type_idx), K(IDX)); } else if (FALSE_IT(new (p_impl) ImplType())) { } else if (MDS_FAIL(p_impl->deserialize(buf, buf_len, pos))) { - MTL(mds::ObTenantMdsService*)->get_buffer_ctx_allocator().free(p_impl); + allocator.free(p_impl); p_impl = nullptr; MDS_LOG(ERROR, "deserialzed from buffer failed", KR(ret), K(type_idx), K(IDX)); } else { @@ -98,7 +98,7 @@ int deserialize_(BufferCtx *&ctx_, int64_t type_idx, const char *buf, const int6 MDS_LOG(INFO, "deserialize ctx success", KR(ret), K(*p_impl), K(type_idx), K(IDX), K(buf_len), K(pos), K(lbt())); } reset_mds_mem_check_thread_local_info(); - } else if (MDS_FAIL(deserialize_(ctx_, type_idx, buf, buf_len, pos))) { + } else if (MDS_FAIL(deserialize_(ctx_, type_idx, buf, buf_len, pos, allocator))) { MDS_LOG(ERROR, "deserialzed from buffer failed", KR(ret), K(type_idx), K(IDX)); } return ret; @@ -109,14 +109,15 @@ int deserialize_(BufferCtx *&ctx_, int64_t type_idx, const char *buf, const int64_t buf_len, - int64_t &pos) + int64_t &pos, + ObIAllocator &allocator) { int ret = OB_ERR_UNEXPECTED; MDS_LOG(ERROR, "type idx out of tuple range", KR(ret), K(type_idx), K(BufferCtxTupleHelper::get_element_size())); return ret; } -int BufferCtxNode::deserialize(const char *buf, const int64_t buf_len, int64_t &pos) +int BufferCtxNode::deserialize(const char *buf, const int64_t buf_len, int64_t &pos, ObIAllocator &allocator) { int ret = OB_SUCCESS; MDS_TG(10_ms); @@ -125,7 +126,7 @@ int BufferCtxNode::deserialize(const char *buf, const int64_t buf_len, int64_t & MDS_LOG(ERROR, "fail to deserialize buffer ctx id", KR(ret), K(type_idx)); } else if (INVALID_VALUE == type_idx) { MDS_LOG(DEBUG, "deserialized INVALD buffer ctx", KR(ret), K(type_idx), K(buf_len), K(pos)); - } else if (MDS_FAIL(deserialize_<0>(ctx_, type_idx, buf, buf_len, pos))) { + } else if (MDS_FAIL(deserialize_<0>(ctx_, type_idx, buf, buf_len, pos, allocator))) { MDS_LOG(WARN, "deserialized buffer ctx failed", KR(ret), K(type_idx)); } return ret; diff --git a/src/storage/multi_data_source/buffer_ctx.h b/src/storage/multi_data_source/buffer_ctx.h index d78ff1a3cd..86e12d8bec 100644 --- a/src/storage/multi_data_source/buffer_ctx.h +++ b/src/storage/multi_data_source/buffer_ctx.h @@ -16,6 +16,7 @@ #include "lib/oblog/ob_log_module.h" #include "runtime_utility/common_define.h" #include "mds_writer.h" +#include "runtime_utility/mds_tenant_service.h" namespace oceanbase { @@ -82,7 +83,10 @@ public: } // 同事务状态一起持久化以及恢复 int serialize(char*, const int64_t, int64_t&) const;// 要把实际的ctx类型编码进二进制中 - int deserialize(const char*, const int64_t, int64_t&);// 要根据实际的ctx的类型，在编译期反射子类类型 + int deserialize(const char*, + const int64_t, + int64_t&, + ObIAllocator &allocator = MTL(ObTenantMdsService*)->get_buffer_ctx_allocator());// 要根据实际的ctx的类型，在编译期反射子类类型 int64_t get_serialize_size(void) const; TO_STRING_KV(KP(this), KP_(ctx), KPC_(ctx)); private: diff --git a/src/storage/multi_data_source/runtime_utility/mds_factory.cpp b/src/storage/multi_data_source/runtime_utility/mds_factory.cpp index 9b2aa363ad..77ab75eb3f 100644 --- a/src/storage/multi_data_source/runtime_utility/mds_factory.cpp +++ b/src/storage/multi_data_source/runtime_utility/mds_factory.cpp @@ -58,6 +58,7 @@ template int deepcopy(const transaction::ObTransID &trans_id, const BufferCtx &old_ctx, BufferCtx *&new_ctx, + ObIAllocator &allocator, const char *alloc_file, const char *alloc_func, const int64_t line) { @@ -74,11 +75,17 @@ int deepcopy(const transaction::ObTransID &trans_id, MDS_ASSERT(OB_NOT_NULL(p_old_impl_ctx)); const ImplType &old_impl_ctx = *p_old_impl_ctx; set_mds_mem_check_thread_local_info(MdsWriter(trans_id), typeid(ImplType).name(), alloc_file, alloc_func, line); - if (CLICK() && - OB_ISNULL(p_impl = (ImplType *)MTL(ObTenantMdsService*)->get_buffer_ctx_allocator().alloc(sizeof(ImplType), - ObMemAttr(MTL_ID(), - "MDS_CTX_COPY", - ObCtxIds::MDS_CTX_ID)))) { + // if pre_alloc buffer_ctx use it + if (OB_NOT_NULL(new_ctx)) { + ImplType *new_ctx_impl = dynamic_cast(new_ctx); + if (MDS_FAIL(common::meta::copy_or_assign(old_impl_ctx, *new_ctx_impl))) { + MDS_LOG(WARN, "fail to assign old ctx to new", KR(ret), K(IDX)); + } + } else if (CLICK() && + OB_ISNULL(p_impl = (ImplType *)allocator.alloc(sizeof(ImplType), + ObMemAttr(MTL_ID(), + "MDS_CTX_COPY", + ObCtxIds::MDS_CTX_ID)))) { ret = OB_ALLOCATE_MEMORY_FAILED; MDS_LOG(WARN, "alloc memory failed", KR(ret), K(IDX)); } else { @@ -86,7 +93,7 @@ int deepcopy(const transaction::ObTransID &trans_id, new (p_impl)ImplType(); if (MDS_FAIL(common::meta::copy_or_assign(old_impl_ctx, *p_impl))) { p_impl->~ImplType(); - MTL(mds::ObTenantMdsService*)->get_buffer_ctx_allocator().free(p_impl); + allocator.free(p_impl); MDS_LOG(WARN, "fail to assign old ctx to new", KR(ret), K(IDX)); } else { new_ctx = p_impl; @@ -95,7 +102,7 @@ int deepcopy(const transaction::ObTransID &trans_id, } reset_mds_mem_check_thread_local_info(); } else { - ret = deepcopy(trans_id, old_ctx, new_ctx, alloc_file, alloc_func, line); + ret = deepcopy(trans_id, old_ctx, new_ctx, allocator, alloc_file, alloc_func, line); } return ret; } @@ -104,6 +111,7 @@ template <> int deepcopy(const transaction::ObTransID &trans_id, const BufferCtx &old_ctx, BufferCtx *&new_ctx, + ObIAllocator &allocator, const char *alloc_file, const char *alloc_func, const int64_t line) @@ -116,6 +124,7 @@ int deepcopy(const transaction::ObTran int MdsFactory::deep_copy_buffer_ctx(const transaction::ObTransID &trans_id, const BufferCtx &old_ctx, BufferCtx *&new_ctx, + ObIAllocator &allocator, const char *alloc_file, const char *alloc_func, const int64_t line) @@ -126,7 +135,7 @@ int MdsFactory::deep_copy_buffer_ctx(const transaction::ObTransID &trans_id, ret = OB_INVALID_ARGUMENT; new_ctx = nullptr;// won't copy MDS_LOG(WARN, "invalid old_ctx", K(old_ctx.get_binding_type_id())); - } else if (MDS_FAIL(deepcopy<0>(trans_id, old_ctx, new_ctx, alloc_file, alloc_func, line))) { + } else if (MDS_FAIL(deepcopy<0>(trans_id, old_ctx, new_ctx, allocator, alloc_file, alloc_func, line))) { MDS_LOG(WARN, "fail to deep copy buffer ctx", K(old_ctx.get_binding_type_id())); } return ret; @@ -149,6 +158,7 @@ void try_set_writer(T &ctx, const transaction::ObTransID &trans_id) { int MdsFactory::create_buffer_ctx(const transaction::ObTxDataSourceType &data_source_type, const transaction::ObTransID &trans_id, BufferCtx *&buffer_ctx, + ObIAllocator &allocator, const char *alloc_file, const char *alloc_func, const int64_t line) { @@ -161,10 +171,10 @@ int MdsFactory::create_buffer_ctx(const transaction::ObTxDataSourceType &data_so set_mds_mem_check_thread_local_info(MdsWriter(trans_id), typeid(BUFFER_CTX_TYPE).name(), alloc_file, alloc_func, line);\ int64_t type_id = TupleTypeIdx::value;\ BUFFER_CTX_TYPE *ctx_impl = (BUFFER_CTX_TYPE *)\ - MTL(ObTenantMdsService*)->get_buffer_ctx_allocator().alloc(sizeof(BUFFER_CTX_TYPE),\ - ObMemAttr(MTL_ID(),\ - "MDS_CTX_CREATE",\ - ObCtxIds::MDS_CTX_ID));\ + allocator.alloc(sizeof(BUFFER_CTX_TYPE),\ + ObMemAttr(MTL_ID(),\ + "MDS_CTX_CREATE",\ + ObCtxIds::MDS_CTX_ID));\ if (OB_ISNULL(ctx_impl)) {\ ret = OB_ALLOCATE_MEMORY_FAILED;\ MDS_LOG(WARN, "alloc memory failed", KR(ret));\ diff --git a/src/storage/multi_data_source/runtime_utility/mds_factory.h b/src/storage/multi_data_source/runtime_utility/mds_factory.h index 94247fbd50..ee364af102 100644 --- a/src/storage/multi_data_source/runtime_utility/mds_factory.h +++ b/src/storage/multi_data_source/runtime_utility/mds_factory.h @@ -91,12 +91,14 @@ struct MdsFactory static int deep_copy_buffer_ctx(const transaction::ObTransID &trans_id, const BufferCtx &old_ctx, BufferCtx *&new_ctx, + ObIAllocator &allocator = MTL(ObTenantMdsService*)->get_buffer_ctx_allocator(), const char *alloc_file = __builtin_FILE(), const char *alloc_func = __builtin_FUNCTION(), const int64_t line = __builtin_LINE()); static int create_buffer_ctx(const transaction::ObTxDataSourceType &data_source_type, const transaction::ObTransID &trans_id, BufferCtx *&buffer_ctx, + ObIAllocator &allocator = MTL(ObTenantMdsService*)->get_buffer_ctx_allocator(), const char *alloc_file = __builtin_FILE(), const char *alloc_func = __builtin_FUNCTION(), const int64_t line = __builtin_LINE()); @@ -144,4 +146,4 @@ private: } } } -#endif \ No newline at end of file +#endif diff --git a/src/storage/ob_i_memtable_mgr.h b/src/storage/ob_i_memtable_mgr.h index 5de698683a..51f60f47cc 100644 --- a/src/storage/ob_i_memtable_mgr.h +++ b/src/storage/ob_i_memtable_mgr.h @@ -259,7 +259,7 @@ public: virtual int get_boundary_memtable(ObTableHandleV2 &handle) { return OB_NOT_SUPPORTED; } - virtual int get_memtable_for_replay(share::SCN replay_scn, ObTableHandleV2 &handle) + virtual int get_memtable_for_replay(const share::SCN &replay_scn, ObTableHandleV2 &handle) { return OB_SUCCESS; } diff --git a/src/storage/ob_i_table.cpp b/src/storage/ob_i_table.cpp index 2e053e060d..e6e938a38a 100644 --- a/src/storage/ob_i_table.cpp +++ b/src/storage/ob_i_table.cpp @@ -76,7 +76,9 @@ const char* ObITable::table_type_name_[] = "DDL_MERGE_CG", "DDL_MEM_CO", "DDL_MEM_CG", - "DDL_MEM_MINI_SSTABLE" + "DDL_MEM_MINI_SSTABLE", + "MDS_MINI", + "MDS_MINOR" }; uint64_t ObITable::TableKey::hash() const diff --git a/src/storage/ob_i_table.h b/src/storage/ob_i_table.h index aa65523530..de28951cb8 100644 --- a/src/storage/ob_i_table.h +++ b/src/storage/ob_i_table.h @@ -106,6 +106,8 @@ public: DDL_MEM_CO_SSTABLE = 23, DDL_MEM_CG_SSTABLE = 24, DDL_MEM_MINI_SSTABLE = 25, + MDS_MINI_SSTABLE = 26, + MDS_MINOR_SSTABLE = 27, // < add new sstable before here, See is_sstable() MAX_TABLE_TYPE diff --git a/src/storage/ob_partition_range_spliter.cpp b/src/storage/ob_partition_range_spliter.cpp index dd80aaf314..92ad5fd7c7 100644 --- a/src/storage/ob_partition_range_spliter.cpp +++ b/src/storage/ob_partition_range_spliter.cpp @@ -1097,37 +1097,20 @@ int ObPartitionRangeSpliter::split_ranges_memtable(ObRangeSplitInfo &range_info, ObSEArray store_ranges; memtable::ObMemtable *memtable = static_cast(table); - if (OB_FAIL(memtable->get_split_ranges( - &range_info.store_range_->get_start_key(), - &range_info.store_range_->get_end_key(), - range_info.parallel_target_count_, - store_ranges))) { - if (OB_ENTRY_NOT_EXIST == ret) { - STORAGE_LOG(WARN, "Failed to get split ranges from memtable, build single range instead", K(ret)); - if (OB_FAIL(build_single_range(false/*for compaction*/, range_info, allocator, range_array))) { - STORAGE_LOG(WARN, "Failed to build single range", K(ret)); - } else { - STORAGE_LOG(DEBUG, "try to make single split range for memtable", K(range_info), K(range_array)); - } - } else { - STORAGE_LOG(WARN, "Failed to get split ranges from memtable", K(ret)); - } + if (OB_ISNULL(memtable)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Unexpected null memtable", K(ret), KP(memtable), K(range_info)); + } else if (OB_FAIL(memtable->get_split_ranges( + *range_info.store_range_, range_info.parallel_target_count_, store_ranges))) { + STORAGE_LOG(WARN, "Failed to get split ranges from memtable", K(ret)); } else { ObStoreRange store_range; for (int64_t i = 0; OB_SUCC(ret) && i < store_ranges.count(); i++) { if (OB_FAIL(store_ranges.at(i).deep_copy(allocator, store_range))) { STORAGE_LOG(WARN, "Failed to deep copy store range", K(ret), K(store_ranges)); } else if (FALSE_IT(store_range.set_table_id(range_info.store_range_->get_table_id()))) { - } else { - if (i == 0 && range_info.store_range_->get_border_flag().inclusive_start()) { - store_range.set_left_closed(); - } - if (i == store_ranges.count() - 1 && range_info.store_range_->get_border_flag().inclusive_end()) { - store_range.set_right_closed(); - } - if (OB_FAIL(range_array.push_back(store_range))) { - STORAGE_LOG(WARN, "Failed to push back store range", K(ret), K(store_range)); - } + } else if (OB_FAIL(range_array.push_back(store_range))) { + STORAGE_LOG(WARN, "Failed to push back store range", K(ret), K(store_range)); } } } diff --git a/src/storage/ob_storage_rpc.cpp b/src/storage/ob_storage_rpc.cpp index 63f1ae4227..89016d985e 100644 --- a/src/storage/ob_storage_rpc.cpp +++ b/src/storage/ob_storage_rpc.cpp @@ -199,7 +199,9 @@ OB_SERIALIZE_MEMBER(ObCopyMacroBlockRangeArg, tenant_id_, ls_id_, table_key_, da ObCopyMacroBlockHeader::ObCopyMacroBlockHeader() : is_reuse_macro_block_(false), - occupy_size_(0) + occupy_size_(0), + macro_meta_row_(), + allocator_("CMBlockHeader") { } @@ -207,6 +209,8 @@ void ObCopyMacroBlockHeader::reset() { is_reuse_macro_block_ = false; occupy_size_ = 0; + macro_meta_row_.reset(); + allocator_.reset(); } bool ObCopyMacroBlockHeader::is_valid() const @@ -2152,6 +2156,8 @@ int ObCheckStartTransferTabletsDelegate::check_transfer_out_tablet_sstable_(cons int ret = OB_SUCCESS; ObTableStoreIterator ddl_iter; ObTabletMemberWrapper wrapper; + const int64_t emergency_sstable_count = ObTabletTableStore::EMERGENCY_SSTABLE_CNT; + if (OB_ISNULL(tablet)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tablet is null", K(ret)); @@ -2159,6 +2165,10 @@ int ObCheckStartTransferTabletsDelegate::check_transfer_out_tablet_sstable_(cons LOG_WARN("fetch table store fail", K(ret), KP(tablet)); } else if (!wrapper.get_member()->get_major_sstables().empty()) { // do nothing + } else if (wrapper.get_member()->get_table_count() > emergency_sstable_count) { + ret = OB_TOO_MANY_SSTABLE; + LOG_WARN("transfer src tablet has too many sstable, cannot transfer, need retry", K(ret), + "table_count", wrapper.get_member()->get_table_count(), "emergency sstable count", emergency_sstable_count); } else if (OB_FAIL(tablet->get_ddl_sstables(ddl_iter))) { LOG_WARN("failed to get ddl sstable", K(ret)); } else if (ddl_iter.is_valid()) { // indicates the existence of ddl sstable diff --git a/src/storage/ob_storage_rpc.h b/src/storage/ob_storage_rpc.h index 6c787e872d..f31bee38fb 100644 --- a/src/storage/ob_storage_rpc.h +++ b/src/storage/ob_storage_rpc.h @@ -114,9 +114,12 @@ public: void reset(); bool is_valid() const; - TO_STRING_KV(K_(is_reuse_macro_block), K_(occupy_size)); + TO_STRING_KV(K_(is_reuse_macro_block), K_(occupy_size), K_(macro_meta_row)); bool is_reuse_macro_block_; int64_t occupy_size_; + + blocksstable::ObDatumRow macro_meta_row_; // used to get macro meta + common::ObArenaAllocator allocator_; }; struct ObCopyTabletInfoArg diff --git a/src/storage/ob_storage_table_guard.cpp b/src/storage/ob_storage_table_guard.cpp index 1391b4a3cc..3e0f0d405b 100644 --- a/src/storage/ob_storage_table_guard.cpp +++ b/src/storage/ob_storage_table_guard.cpp @@ -146,7 +146,7 @@ int ObStorageTableGuard::refresh_and_protect_memtable() int64_t warn_interval = DEFAULT_REFRESH_WARN_INTERVAL; do { - if (OB_FAIL(tablet_->get_boundary_memtable(handle))) { + if (OB_FAIL(tablet_->get_boundary_memtable_from_memtable_mgr(handle))) { // if there is no memtable, create a new one if (OB_ENTRY_NOT_EXIST == ret) { ret = create_data_memtable_(ls_id, tablet_id, need_retry); diff --git a/src/storage/tablelock/ob_mem_ctx_table_lock.cpp b/src/storage/tablelock/ob_mem_ctx_table_lock.cpp index b3dc9de0c4..bb9ac74e1f 100644 --- a/src/storage/tablelock/ob_mem_ctx_table_lock.cpp +++ b/src/storage/tablelock/ob_mem_ctx_table_lock.cpp @@ -377,6 +377,20 @@ int ObLockMemCtx::check_lock_exist( //TODO(lihongqin):check it return ret; } +int ObLockMemCtx::check_contain_tablet(ObTabletID tablet_id, bool &contain) +{ + int ret = OB_SUCCESS; + contain = false; + RDLockGuard guard(list_rwlock_); + DLIST_FOREACH(curr, lock_list_) { + if (curr->lock_op_.is_tablet_lock(tablet_id)) { + contain = true; + break; + } + } + return ret; +} + int ObLockMemCtx::check_modify_schema_elapsed( const ObLockID &lock_id, const int64_t schema_version) diff --git a/src/storage/tablelock/ob_mem_ctx_table_lock.h b/src/storage/tablelock/ob_mem_ctx_table_lock.h index 86baaf5681..63a9d90d36 100644 --- a/src/storage/tablelock/ob_mem_ctx_table_lock.h +++ b/src/storage/tablelock/ob_mem_ctx_table_lock.h @@ -91,6 +91,8 @@ public: const ObTableLockOpType op_type, bool &is_exist, uint64_t lock_mode_cnt_in_same_trans[]) const; + int64_t get_lock_op_count() { return lock_list_.get_size(); } + int check_contain_tablet(ObTabletID tablet_id, bool &contain); // wait all the trans that modify with a smaller schema_version finished. int check_modify_schema_elapsed( const ObLockID &lock_id, diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index 8a9556515b..61b43982f8 100644 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -4282,29 +4282,31 @@ int ObTablet::create_memtable( LOG_WARN("failed to create memtable", K(ret), K(clog_checkpoint_scn), K(schema_version), K(for_replay)); } - } else if (FALSE_IT(time_guard.click("inner_create_memtable"))) { - } else if (OB_FAIL(update_memtables())) { - LOG_WARN("failed to append new memtable to table store", K(ret), KPC(this)); - if (OB_SIZE_OVERFLOW == ret) { - // rewrite errno to OB_EAGAIN when memtable count overflow, in case to stuck the log relpay engine - ret = OB_EAGAIN; - } - } else if (FALSE_IT(time_guard.click("update_memtables"))) { } else { - tablet_addr_.inc_seq(); - table_store_addr_.addr_.inc_seq(); - - if (table_store_addr_.is_memory_object()) { - ObSEArray memtable_array; - if (OB_FAIL(inner_get_memtables(memtable_array, true/*need_active*/))) { - LOG_WARN("inner get memtables fail", K(ret), K(*this)); - } else if (OB_FAIL(table_store_addr_.get_ptr()->update_memtables(memtable_array))) { - LOG_WARN("table store update memtables fail", K(ret), K(memtable_array)); + time_guard.click("inner_create_memtable"); + do { + if (OB_FAIL(update_memtables())) { + LOG_WARN("failed to append new memtable to table store", K(ret), KPC(this)); + } else if (FALSE_IT(time_guard.click("update_memtables"))) { } else { - time_guard.click("ts update mem"); - LOG_INFO("table store update memtable success", K(ret), K(ls_id), K(tablet_id), K_(table_store_addr), KP(this)); + tablet_addr_.inc_seq(); + table_store_addr_.addr_.inc_seq(); + if (table_store_addr_.is_memory_object()) { + ObSEArray memtable_array; + if (OB_FAIL(inner_get_memtables(memtable_array, true/*need_active*/))) { + LOG_WARN("inner get memtables fail", K(ret), K(*this)); + } else if (OB_FAIL(table_store_addr_.get_ptr()->update_memtables(memtable_array))) { + LOG_WARN("table store update memtables fail", K(ret), K(memtable_array)); + } else { + time_guard.click("ts update mem"); + LOG_INFO("table store update memtable success", K(ret), K(ls_id), K(tablet_id), K_(table_store_addr), KP(this)); + } + } } - } + if (OB_FAIL(ret) && REACH_COUNT_INTERVAL(100)) { + LOG_ERROR("fail to refresh tablet memtables, which may cause hang", K(ret), KPC(this)); + } + } while(OB_FAIL(ret)); } STORAGE_LOG(DEBUG, @@ -7359,7 +7361,7 @@ int ObTablet::get_all_memtables(ObTableHdlArray &handle) const return ret; } -int ObTablet::get_boundary_memtable(ObTableHandleV2 &handle) const +int ObTablet::get_boundary_memtable_from_memtable_mgr(ObTableHandleV2 &handle) const { int ret = OB_SUCCESS; ObProtectedMemtableMgrHandle *protected_handle = NULL; diff --git a/src/storage/tablet/ob_tablet.h b/src/storage/tablet/ob_tablet.h index 06da73a143..15e5ff9996 100644 --- a/src/storage/tablet/ob_tablet.h +++ b/src/storage/tablet/ob_tablet.h @@ -347,8 +347,10 @@ public: int update_upper_trans_version(ObLS &ls, bool &is_updated); // memtable operation + // ATTENTION!!! + // - The `get_all_memtables()` is that get all memtables from memtable mgr, not from this tablet. int get_all_memtables(ObTableHdlArray &handle) const; - int get_boundary_memtable(ObTableHandleV2 &handle) const; + int get_boundary_memtable_from_memtable_mgr(ObTableHandleV2 &handle) const; int get_protected_memtable_mgr_handle(ObProtectedMemtableMgrHandle *&handle) const; // get the active memtable for write or replay. diff --git a/src/storage/tablet/ob_tablet_memtable_mgr.cpp b/src/storage/tablet/ob_tablet_memtable_mgr.cpp index ac65b1eb51..f537bf88af 100644 --- a/src/storage/tablet/ob_tablet_memtable_mgr.cpp +++ b/src/storage/tablet/ob_tablet_memtable_mgr.cpp @@ -663,7 +663,7 @@ int ObTabletMemtableMgr::set_is_tablet_freeze_for_active_memtable( return ret; } -int ObTabletMemtableMgr::get_memtable_for_replay(SCN replay_scn, ObTableHandleV2 &handle) +int ObTabletMemtableMgr::get_memtable_for_replay(const SCN &replay_scn, ObTableHandleV2 &handle) { int ret = OB_SUCCESS; ObITabletMemtable *tablet_memtable = nullptr; diff --git a/src/storage/tablet/ob_tablet_memtable_mgr.h b/src/storage/tablet/ob_tablet_memtable_mgr.h index f776799161..c0a696db21 100644 --- a/src/storage/tablet/ob_tablet_memtable_mgr.h +++ b/src/storage/tablet/ob_tablet_memtable_mgr.h @@ -67,7 +67,7 @@ public: // derived from ObIMemtableMgr virtual int get_active_memtable(ObTableHandleV2 &handle) const override; virtual int get_all_memtables(ObTableHdlArray &handle) override; virtual void destroy() override; - virtual int get_memtable_for_replay(share::SCN replay_scn, + virtual int get_memtable_for_replay(const share::SCN &replay_scn, ObTableHandleV2 &handle) override; virtual int get_boundary_memtable(ObTableHandleV2 &handle) override; virtual int create_memtable(const CreateMemtableArg &arg) override; diff --git a/src/storage/tablet/ob_tablet_start_transfer_mds_helper.cpp b/src/storage/tablet/ob_tablet_start_transfer_mds_helper.cpp index 03f082d4f1..cae26ee858 100644 --- a/src/storage/tablet/ob_tablet_start_transfer_mds_helper.cpp +++ b/src/storage/tablet/ob_tablet_start_transfer_mds_helper.cpp @@ -810,6 +810,7 @@ int ObTabletStartTransferOutV2Helper::on_register( ObTxDataSourceType mds_op_type = ObTxDataSourceType::START_TRANSFER_OUT_V2; ObTabletStartTransferOutCommonHelper transfer_out_helper(mds_op_type); bool start_modify = false; + ObTransferOutTxParam param; if (OB_ISNULL(buf) || len < 0) { ret = OB_INVALID_ARGUMENT; @@ -824,11 +825,27 @@ int ObTabletStartTransferOutV2Helper::on_register( } else if (OB_UNLIKELY(nullptr == (ls = ls_handle.get_ls()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls should not be NULL", KR(ret), K(info), KP(ls)); - } else if (OB_FAIL(transfer_tx_ctx.record_transfer_block_op(info.src_ls_id_, info.dest_ls_id_, info.data_end_scn_, info.transfer_epoch_, false))) { + } else if (OB_FAIL(transfer_tx_ctx.record_transfer_block_op(info.src_ls_id_, + info.dest_ls_id_, + info.data_end_scn_, + info.transfer_epoch_, + false, + info.filter_tx_need_transfer_, + info.move_tx_ids_))) { LOG_WARN("record transfer block op failed", KR(ret), K(info)); + } else { + param.except_tx_id_ = user_ctx.get_writer().writer_id_; + param.data_end_scn_ = info.data_end_scn_; + param.op_scn_ = op_scn; + param.op_type_ = NotifyType::REGISTER_SUCC; + param.is_replay_ = false; + param.dest_ls_id_ = info.dest_ls_id_; + param.transfer_epoch_ = info.transfer_epoch_; + param.move_tx_ids_ = &info.move_tx_ids_; + } + if (OB_FAIL(ret)) { } else if (FALSE_IT(start_modify = true)) { - } else if (OB_FAIL(ls->transfer_out_tx_op(user_ctx.get_writer().writer_id_, info.data_end_scn_, op_scn, - NotifyType::REGISTER_SUCC, false, info.dest_ls_id_, info.transfer_epoch_, active_tx_count, block_tx_count))) { + } else if (!info.empty_tx() && OB_FAIL(ls->transfer_out_tx_op(param, active_tx_count, block_tx_count))) { LOG_WARN("transfer block tx failed", KR(ret), K(info)); } else if (OB_FAIL(transfer_out_helper.update_tablets_transfer_out_(info, ls, ctx))) { LOG_WARN("update tablets transfer out failed", KR(ret), K(info), KP(ls)); @@ -840,8 +857,8 @@ int ObTabletStartTransferOutV2Helper::on_register( if (OB_FAIL(ret)) { // to clean int tmp_ret = OB_SUCCESS; - if (start_modify && OB_TMP_FAIL(ls->transfer_out_tx_op(user_ctx.get_writer().writer_id_, info.data_end_scn_, op_scn, - NotifyType::ON_ABORT, false, info.dest_ls_id_, info.transfer_epoch_, active_tx_count, block_tx_count))) { + param.op_type_ = NotifyType::ON_ABORT; + if (start_modify && !info.empty_tx() && OB_TMP_FAIL(ls->transfer_out_tx_op(param, active_tx_count, block_tx_count))) { LOG_ERROR("transfer out clean failed", K(tmp_ret), K(info), K(user_ctx.get_writer().writer_id_)); } } @@ -866,6 +883,7 @@ int ObTabletStartTransferOutV2Helper::on_replay(const char *buf, ObTransferOutTxCtx &transfer_tx_ctx = static_cast(ctx); ObTxDataSourceType mds_op_type = ObTxDataSourceType::START_TRANSFER_OUT_V2; ObTabletStartTransferOutCommonHelper transfer_out_helper(mds_op_type); + ObTransferOutTxParam param; if (OB_ISNULL(buf) || len < 0) { ret = OB_INVALID_ARGUMENT; @@ -880,10 +898,26 @@ int ObTabletStartTransferOutV2Helper::on_replay(const char *buf, } else if (OB_UNLIKELY(nullptr == (ls = ls_handle.get_ls()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls should not be NULL", KR(ret), K(info), KP(ls)); - } else if (OB_FAIL(transfer_tx_ctx.record_transfer_block_op(info.src_ls_id_, info.dest_ls_id_, info.data_end_scn_, info.transfer_epoch_, true))) { + } else if (OB_FAIL(transfer_tx_ctx.record_transfer_block_op(info.src_ls_id_, + info.dest_ls_id_, + info.data_end_scn_, + info.transfer_epoch_, + true, + info.filter_tx_need_transfer_, + info.move_tx_ids_))) { LOG_WARN("record transfer block op failed", KR(ret), K(info)); - } else if (OB_FAIL(ls->transfer_out_tx_op(user_ctx.get_writer().writer_id_, info.data_end_scn_, scn, - NotifyType::ON_REDO, true, info.dest_ls_id_, info.transfer_epoch_, active_tx_count, block_tx_count))) { + } else { + param.except_tx_id_ = user_ctx.get_writer().writer_id_; + param.data_end_scn_ = info.data_end_scn_; + param.op_scn_ = scn; + param.op_type_ = NotifyType::REGISTER_SUCC; + param.is_replay_ = false; + param.dest_ls_id_ = info.dest_ls_id_; + param.transfer_epoch_ = info.transfer_epoch_; + param.move_tx_ids_ = &info.move_tx_ids_; + } + if (OB_FAIL(ret)) { + } else if (!info.empty_tx() && OB_FAIL(ls->transfer_out_tx_op(param, active_tx_count, block_tx_count))) { LOG_WARN("transfer block tx failed", KR(ret), K(info)); } else if (OB_FAIL(transfer_out_helper.on_replay_success_(scn, info, ctx))) { LOG_WARN("start transfer out on replay failed", KR(ret), K(info), KP(ls)); diff --git a/src/storage/tablet/ob_tablet_transfer_tx_ctx.cpp b/src/storage/tablet/ob_tablet_transfer_tx_ctx.cpp index 1a06578da3..f8c3cc4887 100644 --- a/src/storage/tablet/ob_tablet_transfer_tx_ctx.cpp +++ b/src/storage/tablet/ob_tablet_transfer_tx_ctx.cpp @@ -53,13 +53,17 @@ void ObTransferMoveTxParam::reset() is_incomplete_replay_ = false; } -ObTransferOutTxCtx::ObTransferOutTxCtx() - : do_transfer_block_(false), - src_ls_id_(), - dest_ls_id_(), - data_end_scn_(), - transfer_scn_(), - transfer_epoch_(0) {} +void ObTransferOutTxParam::reset() +{ + except_tx_id_ = 0; + data_end_scn_.reset(); + op_scn_.reset(); + op_type_ = NotifyType::UNKNOWN; + is_replay_ = false; + dest_ls_id_.reset(); + transfer_epoch_ = 0; + move_tx_ids_ = nullptr; +} void ObTransferOutTxCtx::reset() { @@ -69,6 +73,8 @@ void ObTransferOutTxCtx::reset() data_end_scn_.reset(); transfer_scn_.reset(); transfer_epoch_ = 0; + filter_tx_need_transfer_ = false; + move_tx_ids_.reset(); } bool ObTransferOutTxCtx::is_valid() @@ -87,6 +93,8 @@ int ObTransferOutTxCtx::assign(const ObTransferOutTxCtx &other) const mds::MdsCtx &mds_ctx = static_cast(other); if (OB_FAIL(MdsCtx::assign(mds_ctx))) { LOG_WARN("transfer out tx ctx assign failed", KR(ret), K(other)); + } else if (OB_FAIL(move_tx_ids_.assign(other.move_tx_ids_))) { + LOG_WARN("assign array failed", KR(ret)); } else { do_transfer_block_ = other.do_transfer_block_; src_ls_id_ = other.src_ls_id_; @@ -94,6 +102,7 @@ int ObTransferOutTxCtx::assign(const ObTransferOutTxCtx &other) data_end_scn_ = other.data_end_scn_; transfer_scn_ = other.transfer_scn_; transfer_epoch_ = other.transfer_epoch_; + filter_tx_need_transfer_ = other.filter_tx_need_transfer_; } return ret; } @@ -102,18 +111,23 @@ int ObTransferOutTxCtx::record_transfer_block_op(const share::ObLSID src_ls_id, const share::ObLSID dest_ls_id, const share::SCN data_end_scn, int64_t transfer_epoch, - bool is_replay) + bool is_replay, + bool filter_tx_need_transfer, + ObIArray &move_tx_ids) { int ret = OB_SUCCESS; if (!is_replay && do_transfer_block_) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ctx do_transfer_block unexpectd", KR(ret), KP(this)); + } else if (OB_FAIL(move_tx_ids_.assign(move_tx_ids))) { + LOG_WARN("assgin array failed", KR(ret)); } else { src_ls_id_ = src_ls_id; dest_ls_id_ = dest_ls_id; data_end_scn_ = data_end_scn; transfer_epoch_ = transfer_epoch; do_transfer_block_ = true; + filter_tx_need_transfer_ = filter_tx_need_transfer; } return ret; } @@ -128,6 +142,19 @@ void ObTransferOutTxCtx::on_redo(const share::SCN &redo_scn) ObLS *ls = nullptr; int64_t active_tx_count = 0; int64_t block_tx_count = 0; + ObTransferOutTxParam param; + param.except_tx_id_ = get_writer().writer_id_; + param.data_end_scn_ = data_end_scn_; + param.op_scn_ = redo_scn; + param.op_type_ = transaction::NotifyType::ON_REDO; + param.is_replay_ = false; + param.dest_ls_id_ = dest_ls_id_; + param.transfer_epoch_ = transfer_epoch_; + if (filter_tx_need_transfer_) { + param.move_tx_ids_ = &move_tx_ids_; + } else { + param.move_tx_ids_ = nullptr; + } while (true) { int ret = OB_SUCCESS; @@ -139,15 +166,7 @@ void ObTransferOutTxCtx::on_redo(const share::SCN &redo_scn) } else if (OB_UNLIKELY(nullptr == (ls = ls_handle.get_ls()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls should not be NULL", KR(ret), KP(this), KP(ls)); - } else if (OB_FAIL(ls->transfer_out_tx_op(get_writer().writer_id_, - data_end_scn_, - redo_scn, - transaction::NotifyType::ON_REDO, - false, - dest_ls_id_, - transfer_epoch_, - active_tx_count, - block_tx_count))) { + } else if (!empty_tx() && OB_FAIL(ls->transfer_out_tx_op(param, active_tx_count, block_tx_count))) { LOG_WARN("transfer out tx failed", KR(ret), K(tx_id), KP(this)); } if (OB_FAIL(ret)) { @@ -164,6 +183,19 @@ void ObTransferOutTxCtx::on_commit(const share::SCN &commit_version, const share LOG_INFO("transfer_out_tx on_commit", K(commit_version), K(commit_scn), K(tx_id), KP(this), KPC(this)); int ret = OB_SUCCESS; mds::MdsCtx::on_commit(commit_version, commit_scn); + ObTransferOutTxParam param; + param.except_tx_id_ = get_writer().writer_id_; + param.data_end_scn_ = data_end_scn_; + param.op_scn_ = commit_scn; + param.op_type_ = transaction::NotifyType::ON_COMMIT; + param.is_replay_ = false; + param.dest_ls_id_ = dest_ls_id_; + param.transfer_epoch_ = transfer_epoch_; + if (filter_tx_need_transfer_) { + param.move_tx_ids_ = &move_tx_ids_; + } else { + param.move_tx_ids_ = nullptr; + } while (true) { int ret = OB_SUCCESS; ObLSHandle ls_handle; @@ -180,15 +212,7 @@ void ObTransferOutTxCtx::on_commit(const share::SCN &commit_version, const share } else if (OB_UNLIKELY(nullptr == (ls = ls_handle.get_ls()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls should not be NULL", KR(ret), KP(this)); - } else if (OB_FAIL(ls->transfer_out_tx_op(get_writer().writer_id_, - data_end_scn_, - commit_scn, - transaction::NotifyType::ON_COMMIT, - false, - dest_ls_id_, - transfer_epoch_, - active_tx_count, - op_tx_count))) { + } else if (!empty_tx() && OB_FAIL(ls->transfer_out_tx_op(param, active_tx_count, op_tx_count))) { LOG_WARN("transfer out tx op failed", KR(ret), K(tx_id), KP(this)); } else { int64_t end_time = ObTimeUtility::current_time(); @@ -211,6 +235,19 @@ void ObTransferOutTxCtx::on_abort(const share::SCN &abort_scn) transaction::ObTransID tx_id = writer_.writer_id_; LOG_INFO("transfer_out_tx on_abort", K(abort_scn), K(tx_id), KP(this), KPC(this)); mds::MdsCtx::on_abort(abort_scn); + ObTransferOutTxParam param; + param.except_tx_id_ = get_writer().writer_id_; + param.data_end_scn_ = data_end_scn_; + param.op_scn_ = abort_scn; + param.op_type_ = transaction::NotifyType::ON_ABORT; + param.is_replay_ = false; + param.dest_ls_id_ = dest_ls_id_; + param.transfer_epoch_ = transfer_epoch_; + if (filter_tx_need_transfer_) { + param.move_tx_ids_ = &move_tx_ids_; + } else { + param.move_tx_ids_ = nullptr; + } if (do_transfer_block_) { while (true) { int ret = OB_SUCCESS; @@ -227,15 +264,7 @@ void ObTransferOutTxCtx::on_abort(const share::SCN &abort_scn) } else if (OB_UNLIKELY(nullptr == (ls = ls_handle.get_ls()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls should not be NULL", KR(ret), KP(this)); - } else if (OB_FAIL(ls->transfer_out_tx_op(get_writer().writer_id_, - data_end_scn_, - abort_scn, - transaction::NotifyType::ON_ABORT, - false, - dest_ls_id_, - transfer_epoch_, - active_tx_count, - op_tx_count))) { + } else if (!empty_tx() && OB_FAIL(ls->transfer_out_tx_op(param, active_tx_count, op_tx_count))) { LOG_WARN("transfer out tx op failed", KR(ret), K(tx_id), KP(this)); } if (OB_SUCC(ret)) { diff --git a/src/storage/tablet/ob_tablet_transfer_tx_ctx.h b/src/storage/tablet/ob_tablet_transfer_tx_ctx.h index a777f9be62..c1344b11db 100644 --- a/src/storage/tablet/ob_tablet_transfer_tx_ctx.h +++ b/src/storage/tablet/ob_tablet_transfer_tx_ctx.h @@ -80,6 +80,23 @@ public: bool is_incomplete_replay_; }; +struct ObTransferOutTxParam +{ + ObTransferOutTxParam() { reset(); } + ~ObTransferOutTxParam() { reset(); } + void reset(); + TO_STRING_KV(K_(except_tx_id), K_(data_end_scn), K_(op_scn), K_(op_type), + K_(is_replay), K_(dest_ls_id), K_(transfer_epoch), K_(move_tx_ids)); + int64_t except_tx_id_; + share::SCN data_end_scn_; + share::SCN op_scn_; + transaction::NotifyType op_type_; + bool is_replay_; + share::ObLSID dest_ls_id_; + int64_t transfer_epoch_; + ObIArray *move_tx_ids_; +}; + struct CollectTxCtxInfo final { OB_UNIS_VERSION(1); @@ -145,25 +162,30 @@ class ObTransferOutTxCtx : public mds::MdsCtx { OB_UNIS_VERSION(1); public: - ObTransferOutTxCtx(); + ObTransferOutTxCtx() { reset(); } ~ObTransferOutTxCtx() { reset(); } void reset(); int record_transfer_block_op(const share::ObLSID src_ls_id, const share::ObLSID dest_ls_id, const share::SCN data_end_scn, int64_t transfer_epoch, - bool is_replay); + bool is_replay, + bool filter_tx_need_transfer, + ObIArray &move_tx_ids); virtual void on_redo(const share::SCN &redo_scn) override; virtual void on_commit(const share::SCN &commit_version, const share::SCN &commit_scn) override; virtual void on_abort(const share::SCN &abort_scn) override; bool is_valid(); int assign(const ObTransferOutTxCtx &other); + bool empty_tx() { return filter_tx_need_transfer_ && move_tx_ids_.count() == 0; } TO_STRING_KV(K_(do_transfer_block), K_(src_ls_id), K_(dest_ls_id), K_(data_end_scn), - K_(transfer_scn)); + K_(transfer_scn), + K_(filter_tx_need_transfer), + K_(move_tx_ids)); private: bool do_transfer_block_; share::ObLSID src_ls_id_; @@ -171,6 +193,8 @@ private: share::SCN data_end_scn_; share::SCN transfer_scn_; int64_t transfer_epoch_; + bool filter_tx_need_transfer_; + ObSEArray move_tx_ids_; }; OB_SERIALIZE_MEMBER_TEMP(inline, ObTransferOutTxCtx, @@ -180,7 +204,9 @@ OB_SERIALIZE_MEMBER_TEMP(inline, ObTransferOutTxCtx, dest_ls_id_, data_end_scn_, transfer_scn_, - transfer_epoch_) + transfer_epoch_, + filter_tx_need_transfer_, + move_tx_ids_) class ObTransferMoveTxCtx : public mds::BufferCtx { diff --git a/src/storage/tx/ob_ctx_tx_data.cpp b/src/storage/tx/ob_ctx_tx_data.cpp index 076b2aae3c..758107c5ce 100644 --- a/src/storage/tx/ob_ctx_tx_data.cpp +++ b/src/storage/tx/ob_ctx_tx_data.cpp @@ -132,28 +132,6 @@ int ObCtxTxData::recover_tx_data(ObTxData *tmp_tx_data) return ret; } -int ObCtxTxData::deep_copy_tx_data_out(ObTxDataGuard &tmp_tx_data_guard) -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - - if (OB_FAIL(check_tx_data_writable_())) { - TRANS_LOG(WARN, "tx data is not writeable", K(ret), K(*this)); - } else { - ObTxTable *tx_table = nullptr; - GET_TX_TABLE_(tx_table) - if (OB_FAIL(ret)) { - } else if (OB_FAIL(deep_copy_tx_data_(tx_table, tmp_tx_data_guard))) { - TRANS_LOG(WARN, "deep copy tx data failed", K(ret), K(tmp_tx_data_guard), K(*this)); - } else if (OB_ISNULL(tmp_tx_data_guard.tx_data())) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "copied tmp tx data is null", KR(ret), K(*this)); - } - } - - return ret; -} - int ObCtxTxData::free_tmp_tx_data(ObTxData *&tmp_tx_data) { int ret = OB_SUCCESS; @@ -212,6 +190,38 @@ int ObCtxTxData::set_state(int32_t state) return ret; } +int ObCtxTxData::add_abort_op(SCN op_scn) +{ + int ret = OB_SUCCESS; + RLockGuard guard(lock_); + + ObTxOp abort_op; + if (OB_FAIL(check_tx_data_writable_())) { + TRANS_LOG(WARN, "tx data is not writeable", KR(ret), K(*this)); + } else if (OB_FAIL(tx_data_guard_.tx_data()->init_tx_op())) { + TRANS_LOG(WARN, "init_tx_op failed", KR(ret)); + } else if (OB_FAIL(abort_op.init(ObTxOpCode::ABORT_OP, op_scn, &DEFAULT_TX_DUMMY_OP, 0))) { + TRANS_LOG(WARN, "init_tx_op failed", KR(ret)); + } else if (OB_FAIL(tx_data_guard_.tx_data()->op_guard_->add_tx_op(abort_op))) { + TRANS_LOG(WARN, "add_tx_op failed", KR(ret)); + } + return ret; +} + +int ObCtxTxData::reserve_tx_op_space(int64_t count) +{ + int ret = OB_SUCCESS; + RLockGuard guard(lock_); + if (OB_FAIL(check_tx_data_writable_())) { + TRANS_LOG(WARN, "tx data is not writeable", KR(ret), K(*this)); + } else if (OB_FAIL(tx_data_guard_.tx_data()->init_tx_op())) { + TRANS_LOG(WARN, "init_tx_op failed", KR(ret)); + } else if (OB_FAIL(tx_data_guard_.tx_data()->op_guard_->reserve_tx_op_space(count))) { + TRANS_LOG(WARN, "reserve tx_op space failed", KR(ret)); + } + return ret; +} + int ObCtxTxData::set_commit_version(const SCN &commit_version) { int ret = OB_SUCCESS; @@ -341,80 +351,6 @@ int ObCtxTxData::get_tx_data_ptr(storage::ObTxData *&tx_data_ptr) return ret; } -int ObCtxTxData::prepare_add_undo_action(ObUndoAction &undo_action, - storage::ObTxDataGuard &tmp_tx_data_guard, - storage::ObUndoStatusNode *&tmp_undo_status) -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - /* - * alloc undo_status_node used on commit stage - * alloc tx_data and add undo_action to it, which will be inserted - * into tx_data_table after RollbackSavepoint log sync success - */ - if (OB_FAIL(check_tx_data_writable_())) { - TRANS_LOG(WARN, "tx data is not writeable", K(ret), K(*this)); - } else { - ObTxTable *tx_table = nullptr; - GET_TX_TABLE_(tx_table); - if (OB_FAIL(ret)) { - } else if (OB_FAIL(tx_table->get_tx_data_table()->alloc_undo_status_node(tmp_undo_status))) { - TRANS_LOG(WARN, "alloc undo status fail", K(ret), KPC(this)); - } else if (OB_ISNULL(tmp_undo_status)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "undo status is null", KR(ret), KPC(this)); - } else if (OB_FAIL(tx_table->deep_copy_tx_data(tx_data_guard_, tmp_tx_data_guard))) { - TRANS_LOG(WARN, "copy tx data fail", K(ret), KPC(this)); - } else if (OB_ISNULL(tmp_tx_data_guard.tx_data())) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "copied tx_data is null", KR(ret), KPC(this)); - } else if (OB_FAIL(tmp_tx_data_guard.tx_data()->add_undo_action(tx_table, undo_action))) { - TRANS_LOG(WARN, "add undo action fail", K(ret), KPC(this)); - } - - if (OB_FAIL(ret) && OB_NOT_NULL(tmp_undo_status)) { - tx_table->get_tx_data_table()->free_undo_status_node(tmp_undo_status); - } - } - return ret; -} - -int ObCtxTxData::cancel_add_undo_action(storage::ObUndoStatusNode *tmp_undo_status) -{ - int ret = OB_SUCCESS; - ObTxTable *tx_table = nullptr; - GET_TX_TABLE_(tx_table); - if (OB_SUCC(ret)) { - ret = tx_table->get_tx_data_table()->free_undo_status_node(tmp_undo_status); - } - return ret; -} - -int ObCtxTxData::commit_add_undo_action(ObUndoAction &undo_action, storage::ObUndoStatusNode *tmp_undo_status) -{ - return add_undo_action(undo_action, tmp_undo_status); -} - -int ObCtxTxData::add_undo_action(ObUndoAction &undo_action, storage::ObUndoStatusNode *tmp_undo_status) -{ - int ret = OB_SUCCESS; - RLockGuard guard(lock_); - - if (OB_FAIL(check_tx_data_writable_())) { - TRANS_LOG(WARN, "tx data is not writeable", K(ret), K(*this)); - } else { - ObTxTable *tx_table = nullptr; - GET_TX_TABLE_(tx_table); - if (OB_FAIL(ret)) { - // do nothing - } else if (OB_FAIL(tx_data_guard_.tx_data()->add_undo_action(tx_table, undo_action, tmp_undo_status))) { - TRANS_LOG(WARN, "add undo action failed", K(ret), K(undo_action), KP(tmp_undo_status), K(*this)); - }; - } - - return ret; -} - int ObCtxTxData::check_tx_data_writable_() { int ret = OB_SUCCESS; diff --git a/src/storage/tx/ob_ctx_tx_data.h b/src/storage/tx/ob_ctx_tx_data.h index 6e158b523b..3df982fa70 100644 --- a/src/storage/tx/ob_ctx_tx_data.h +++ b/src/storage/tx/ob_ctx_tx_data.h @@ -51,9 +51,11 @@ public: void get_tx_table(storage::ObTxTable *&tx_table); int set_state(int32_t state); + int add_abort_op(share::SCN op_scn); int set_commit_version(const share::SCN &commit_version); int set_start_log_ts(const share::SCN &start_ts); int set_end_log_ts(const share::SCN &end_ts); + int reserve_tx_op_space(int64_t count); int32_t get_state() const; const share::SCN get_commit_version() const; @@ -62,13 +64,6 @@ public: ObTransID get_tx_id() const; - int prepare_add_undo_action(ObUndoAction &undo_action, - storage::ObTxDataGuard &tmp_tx_data_guard, - storage::ObUndoStatusNode *&tmp_undo_status); - int cancel_add_undo_action(storage::ObUndoStatusNode *tmp_undo_status); - int commit_add_undo_action(ObUndoAction &undo_action, storage::ObUndoStatusNode *tmp_undo_status); - int add_undo_action(ObUndoAction &undo_action, storage::ObUndoStatusNode *tmp_undo_status = NULL); - int get_tx_data(storage::ObTxDataGuard &tx_data_guard); // ATTENTION : use get_tx_data_ptr only if you can make sure the life cycle of ctx_tx_data is longer than your usage diff --git a/src/storage/tx/ob_multi_data_source.cpp b/src/storage/tx/ob_multi_data_source.cpp index 5465be1aab..59462a4998 100644 --- a/src/storage/tx/ob_multi_data_source.cpp +++ b/src/storage/tx/ob_multi_data_source.cpp @@ -26,6 +26,7 @@ #include "storage/multi_data_source/compile_utility/mds_register.h" #undef NEED_MDS_REGISTER_DEFINE #include "share/ob_standby_upgrade.h" // ObStandbyUpgrade +#include "share/allocator/ob_shared_memory_allocator_mgr.h" namespace oceanbase { @@ -105,6 +106,144 @@ bool ObTxBufferNode::operator==(const ObTxBufferNode &buffer_node) const return is_same; } +ObTxBufferNodeWrapper::~ObTxBufferNodeWrapper() +{ + ObIAllocator &allocator = MTL(share::ObSharedMemAllocMgr*)->tx_data_op_allocator(); + if (OB_NOT_NULL(node_.get_ptr())) { + allocator.free(node_.get_ptr()); + } + storage::mds::BufferCtx *buffer_ctx = const_cast(node_.get_buffer_ctx_node().get_ctx()); + if (OB_NOT_NULL(buffer_ctx)) { + // TODO destructor without allocator is safe? + buffer_ctx->~BufferCtx(); + allocator.free(buffer_ctx); + } +} + +OB_DEF_SERIALIZE_SIZE(ObTxBufferNodeWrapper) +{ + int64_t len = 0; + len += serialization::encoded_length_vi64(tx_id_); + len += node_.get_serialize_size(); + len += node_.get_buffer_ctx_node().get_serialize_size(); + return len; +} + +OB_DEF_SERIALIZE(ObTxBufferNodeWrapper) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(serialization::encode_vi64(buf, buf_len, pos, tx_id_))) { + TRANS_LOG(WARN, "serialize node wrapper fail", KR(ret), K(buf_len), K(pos)); + } else if (OB_FAIL(node_.serialize(buf, buf_len, pos))) { + TRANS_LOG(WARN, "serialize node wrapper fail", KR(ret), K(buf_len), K(pos)); + } else if (OB_FAIL(node_.get_buffer_ctx_node().serialize(buf, buf_len, pos))) { + TRANS_LOG(WARN, "serialize node wrapper fail", KR(ret), K(buf_len), K(pos)); + } + return ret; +} + +OB_DEF_DESERIALIZE(ObTxBufferNodeWrapper) +{ + int ret = OB_SUCCESS; + ObIAllocator &allocator = MTL(share::ObSharedMemAllocMgr*)->tx_data_op_allocator(); + char *node_buf = NULL; + if (OB_FAIL(serialization::decode_vi64(buf, data_len, pos, &tx_id_))) { + TRANS_LOG(WARN, "deserialize node wrapper fail", KR(ret), K(data_len), K(pos)); + } else if (OB_FAIL(node_.deserialize(buf, data_len, pos))) { + TRANS_LOG(WARN, "deserialize node wrapper fail", KR(ret), K(data_len), K(pos)); + } else if (OB_ISNULL(node_buf = (char*)allocator.alloc(node_.get_data_size()))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + TRANS_LOG(WARN, "deserialize node wrapper fail", KR(ret), K(data_len), K(pos)); + } else if (FALSE_IT(MEMCPY(node_buf, node_.get_ptr(), node_.get_data_size()))) { + } else if (FALSE_IT((node_.get_data().assign_ptr(node_buf, node_.get_data_size())))) { + } else if (OB_FAIL(node_.get_buffer_ctx_node().deserialize(buf, data_len, pos, allocator))) { + TRANS_LOG(WARN, "deserialize node wrapper fail", KR(ret), K(data_len), K(pos)); + } + return ret; +} + +int ObTxBufferNodeWrapper::assign(ObIAllocator &allocator, const ObTxBufferNodeWrapper &wrapper) +{ + return assign(wrapper.get_tx_id(), wrapper.get_node(), allocator, false); +} + +int ObTxBufferNodeWrapper::pre_alloc(int64_t tx_id, const ObTxBufferNode &node, ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + int64_t buf_len = node.get_data_size(); + char *ptr = NULL; + node_.register_no_ = node.register_no_; + node_.type_ = node.type_; + + if (OB_ISNULL(ptr = (char*)allocator.alloc(buf_len))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + TRANS_LOG(WARN, "alloc mem fail", K(ret)); + } else { + node_.get_data().assign_ptr(ptr, buf_len); + } + + if (OB_SUCC(ret)) { + mds::BufferCtx *new_ctx = nullptr; + if (OB_ISNULL(node.get_buffer_ctx_node().get_ctx())) { + // do nothing + } else if (OB_FAIL(mds::MdsFactory::create_buffer_ctx(node.type_, + ObTransID(tx_id), + new_ctx, + allocator))) { + TRANS_LOG(WARN, "create buffer_ctx failed", KR(ret)); + } else { + node_.get_buffer_ctx_node().set_ctx(new_ctx); + } + } + return ret; +} + +int ObTxBufferNodeWrapper::assign(int64_t tx_id, + const ObTxBufferNode &node, + ObIAllocator &allocator, + bool has_pre_alloc) +{ + int ret = OB_SUCCESS; + int64_t buf_len = node.get_data_size(); + char *ptr = NULL; + tx_id_ = tx_id; + node_.register_no_ = node.register_no_; + node_.has_submitted_ = node.has_submitted_; + node_.has_synced_ = node.has_synced_; + node_.mds_base_scn_ = node.mds_base_scn_; + node_.type_ = node.type_; + + if (has_pre_alloc) { + MEMCPY(node_.get_ptr(), const_cast(node).get_ptr(), buf_len); + } else if (OB_ISNULL(ptr = (char*)allocator.alloc(buf_len))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + TRANS_LOG(WARN, "alloc mem fail", K(ret)); + } else { + MEMCPY(ptr, const_cast(node).get_ptr(), buf_len); + node_.get_data().assign_ptr(ptr, buf_len); + } + + if (OB_SUCC(ret)) { + mds::BufferCtx *new_ctx = nullptr; + if (OB_ISNULL(node.get_buffer_ctx_node().get_ctx())) { + // do nothing + } else if (has_pre_alloc && FALSE_IT(new_ctx = const_cast(node_.get_buffer_ctx_node().get_ctx()))) { + } else if (has_pre_alloc && OB_ISNULL(new_ctx)) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "has_pre_alloc but new_ctx is null", KR(ret), K(node_)); + } else if (OB_FAIL(mds::MdsFactory::deep_copy_buffer_ctx(ObTransID(tx_id_), + *(node.get_buffer_ctx_node().get_ctx()), + new_ctx, + allocator))) { + TRANS_LOG(WARN, "copy buffer_ctx failed", KR(ret)); + } else if (!has_pre_alloc) { + node_.get_buffer_ctx_node().set_ctx(new_ctx); + } + } + return ret; +} + + //##################################################### // ObMulSourceTxDataNotifier //##################################################### diff --git a/src/storage/tx/ob_multi_data_source.h b/src/storage/tx/ob_multi_data_source.h index 5fc535d1d8..2d7e7f0278 100644 --- a/src/storage/tx/ob_multi_data_source.h +++ b/src/storage/tx/ob_multi_data_source.h @@ -84,6 +84,7 @@ class ObTxBufferNode friend class ObTxExecInfo; friend class ObMulSourceTxDataNotifier; friend class ObTxMDSCache; + friend class ObTxBufferNodeWrapper; OB_UNIS_VERSION(1); public: @@ -116,10 +117,11 @@ public: // only for some mds types of CDC // can not be used by observer functions - bool allow_to_use_mds_big_segment() { return type_ == ObTxDataSourceType::DDL_TRANS; } + bool allow_to_use_mds_big_segment() const { return type_ == ObTxDataSourceType::DDL_TRANS; } void replace_data(const common::ObString &data); + ObString &get_data() { return data_; } int64_t get_data_size() const { return data_.length(); } ObTxDataSourceType get_data_source_type() const { return type_; } const ObString &get_data_buf() const { return data_; } @@ -141,6 +143,7 @@ public: has_synced_ = false; } storage::mds::BufferCtxNode &get_buffer_ctx_node() const { return buffer_ctx_node_; } + TO_STRING_KV(K(register_no_), K(has_submitted_), K(has_synced_), K_(type), K(data_.length())); private: @@ -156,6 +159,29 @@ private: typedef common::ObSEArray ObTxBufferNodeArray; typedef common::ObSEArray ObTxBufferCtxArray; +// manage mds_op contain (buffer_node, buffer, buffer_ctx) +class ObTxBufferNodeWrapper +{ + OB_UNIS_VERSION(1); +public: + ObTxBufferNodeWrapper() : tx_id_(0), node_() + {} + ObTxBufferNodeWrapper(const ObTxBufferNodeWrapper &) = delete; + ObTxBufferNodeWrapper &operator=(const ObTxBufferNodeWrapper &) = delete; + ~ObTxBufferNodeWrapper(); + const ObTxBufferNode &get_node() const { return node_; } + int64_t get_tx_id() const { return tx_id_; } + int pre_alloc(int64_t tx_id, const ObTxBufferNode &node, ObIAllocator &allocator); + // deep_copy by node + int assign(int64_t tx_id, const ObTxBufferNode &node, ObIAllocator &allocator, bool has_pre_alloc); + int assign(ObIAllocator &allocator, const ObTxBufferNodeWrapper &node_wrapper); + + TO_STRING_KV(K_(tx_id), K_(node)); +private: + int64_t tx_id_; + ObTxBufferNode node_; +}; + class ObMulSourceTxDataNotifier { public: @@ -190,7 +216,7 @@ private: class ObMulSourceTxDataDump { -public: +public: static const char* dump_buf(ObTxDataSourceType source_type, const char * buf,const int64_t len); private: diff --git a/src/storage/tx/ob_trans_ctx_mgr_v4.cpp b/src/storage/tx/ob_trans_ctx_mgr_v4.cpp index 34f81d711b..af6e255359 100644 --- a/src/storage/tx/ob_trans_ctx_mgr_v4.cpp +++ b/src/storage/tx/ob_trans_ctx_mgr_v4.cpp @@ -10,6 +10,8 @@ * See the Mulan PubL v2 for more details. */ +#define USING_LOG_PREFIX TRANS + #include "lib/hash/ob_hashmap.h" #include "lib/worker.h" #include "lib/list/ob_list.h" @@ -38,6 +40,12 @@ using namespace observer; namespace transaction { +#ifdef ERRSIM +ERRSIM_POINT_DEF(EN_SWITCH_TO_FOLLOWER_GRACEFULLY) +ERRSIM_POINT_DEF(EN_SUBMIT_START_WORKING_LOG) +ERRSIM_POINT_DEF(EN_APPLY_START_WORKING_LOG) +#endif + void ObLSTxCtxIterator::reset() { is_ready_ = false; current_bucket_pos_ = -1; @@ -180,10 +188,13 @@ int ObLSTxCtxMgr::init(const int64_t tenant_id, ret = OB_ERR_UNEXPECTED; } else if (OB_FAIL(ls_log_writer_.init(tenant_id, ls_id, tx_log_adapter_, this))) { TRANS_LOG(WARN, "ls_log_writer init fail", KR(ret)); + } else if (OB_FAIL(tx_ls_state_mgr_.init(ls_id))) { + TRANS_LOG(WARN, "init tx_ls_state_mgr_ failed", KR(ret)); + } else if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::START))) { + TRANS_LOG(WARN, "start ls_tx_ctx_mgr failed",K(ret),K(tx_ls_state_mgr_)); } else { is_inited_ = true; is_leader_serving_ = false; - state_ = State::F_ALL_BLOCKED; tenant_id_ = tenant_id; ls_id_ = ls_id; tx_table_ = tx_table; @@ -214,7 +225,6 @@ void ObLSTxCtxMgr::destroy() void ObLSTxCtxMgr::reset() { is_inited_ = false; - state_ = State::INVALID; tenant_id_ = 0; ls_id_.reset(); tx_table_ = NULL; @@ -230,6 +240,7 @@ void ObLSTxCtxMgr::reset() online_ts_ = 0; txs_ = NULL; ts_mgr_ = NULL; + tx_ls_state_mgr_.reset(); ls_retain_ctx_mgr_.reset(); ObRemoveAllTxCtxFunctor fn; @@ -272,112 +283,6 @@ void ObLSTxCtxMgr::print_all_tx_ctx_(const int64_t max_print, const bool verbose ls_tx_ctx_map_.for_each(print_fn); } -// NB: BLOCK_NORMAL only block noraml tran, only leader can block normal trans -//STATE \ ACTION START LEADER_REVOKE SWL_CB_SUCC SWL_CB_FAIL LEADER_TAKEOVER RESUME_LEADER BLOCK_TX BLOCK_NORMAL BLOCK_ALL STOP ONLINE UNBLOCK_NORMAL -//----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -//INIT F_WORKING N N N N N N N N N N N -//F_WORKING N F_WORKING N N T_PENDING R_PENDING F_TX_BLOCKED N F_ALL_BLOCKED STOPPED N N -//T_PENDING N F_WORKING L_WORKING T_PENDING N N T_TX_BLOCKED_PENDING N T_ALL_BLOCKED_PENDING STOPPED N N -//R_PENDING N F_WORKING L_WORKING R_PENDING N N R_TX_BLOCKED_PENDING N R_ALL_BLOCKED_PENDING STOPPED N N -//L_WORKING N F_WORKING N N N N L_TX_BLOCKED L_BLOCKED_NORMAL L_ALL_BLOCKED STOPPED N N -//F_TX_BLOCKED N F_TX_BLOCKED N N T_TX_BLOCKED_PENDING R_TX_BLOCKED_PENDING F_TX_BLOCKED N F_ALL_BLOCKED STOPPED F_WORKING N -//L_TX_BLOCKED N F_TX_BLOCKED N N N N L_TX_BLOCKED N L_ALL_BLOCKED STOPPED N N -//L_BLOCKED_NORMAL N F_WORKING N N N N L_TX_BLOCKED L_BLOCKED_NORMAL L_ALL_BLOCKED STOPPED N L_WORKING -// -//T_TX_BLOCKED_PENDING N F_TX_BLOCKED L_TX_BLOCKED T_TX_BLOCKED_PENDING N N T_TX_BLOCKED_PENDING N T_ALL_BLOCKED_PENDING STOPPED N N -//R_TX_BLOCKED_PENDING N F_TX_BLOCKED L_TX_BLOCKED R_TX_BLOCKED_PENDING N N R_TX_BLOCKED_PENDING N R_ALL_BLOCKED_PENDING STOPPED N N -// -//F_ALL_BLOCKED N F_ALL_BLOCKED N N T_ALL_BLOCKED_PENDING R_ALL_BLOCKED_PENDING F_ALL_BLOCKED N F_ALL_BLOCKED STOPPED N N -//T_ALL_BLOCKED_PENDING N F_ALL_BLOCKED L_ALL_BLOCKED T_ALL_BLOCKED_PENDING N N T_ALL_BLOCKED_PENDING N T_ALL_BLOCKED_PENDING STOPPED N N -//R_ALL_BLOCKED_PENDING N F_ALL_BLOCKED L_ALL_BLOCKED R_ALL_BLOCKED_PENDING N N R_ALL_BLOCKED_PENDING N R_ALL_BLOCKED_PENDING STOPPED N N -//L_ALL_BLOCKED N F_ALL_BLOCKED N N N N L_ALL_BLOCKED N L_ALL_BLOCKED STOPPED N N -// -//STOPPED N STOPPED N N N N N N N STOPPED N N -//END N N N N N N N N N N N N - -// return value: -// OB_SUCCESS for success -// OB_STATE_NOT_MATCH for switch state fail -// others for error -int ObLSTxCtxMgr::StateHelper::switch_state(const int64_t op) -{ - int ret = OB_SUCCESS; - static const int64_t N = State::INVALID; - static const int64_t F_WORKING = State::F_WORKING; - static const int64_t L_WORKING = State::L_WORKING; - static const int64_t T_PENDING = State::T_PENDING; - static const int64_t R_PENDING = State::R_PENDING; - static const int64_t L_TX_BLOCKED = State::L_TX_BLOCKED; - static const int64_t F_TX_BLOCKED = State::F_TX_BLOCKED; - static const int64_t T_TX_BLOCKED_PENDING = State::T_TX_BLOCKED_PENDING; - static const int64_t R_TX_BLOCKED_PENDING = State::R_TX_BLOCKED_PENDING; - static const int64_t L_BLOCKED_NORMAL = State::L_BLOCKED_NORMAL; - - static const int64_t F_ALL_BLOCKED = State::F_ALL_BLOCKED; - static const int64_t T_ALL_BLOCKED_PENDING = State::T_ALL_BLOCKED_PENDING; - static const int64_t R_ALL_BLOCKED_PENDING = State::R_ALL_BLOCKED_PENDING; - static const int64_t L_ALL_BLOCKED = State::L_ALL_BLOCKED; - - static const int64_t STOPPED = State::STOPPED; - static const int64_t END = State::END; - - static const int64_t STATE_MAP[State::MAX][Ops::MAX] = { -// START L_REVOKE SWL_CB_SUCC SWL_CB_FAIL LEADER_TAKEOVER RESUME_LEADER BLOCK_TX BLOCK_NORMAL BLOCK_ALL STOP ONLINE UNBLOCK_NORMAL - {F_WORKING, N, N, N, N, N, N, N, N, N, N, N}, - {N, F_WORKING, N, N, T_PENDING, R_PENDING, F_TX_BLOCKED, N, F_ALL_BLOCKED, STOPPED, N, N}, - {N, F_WORKING, L_WORKING, T_PENDING, N, N, T_TX_BLOCKED_PENDING, N, T_ALL_BLOCKED_PENDING, STOPPED, N, N}, - {N, F_WORKING, L_WORKING, R_PENDING, N, N, R_TX_BLOCKED_PENDING, N, R_ALL_BLOCKED_PENDING, STOPPED, N, N}, - {N, F_WORKING, N, N, N, N, L_TX_BLOCKED, L_BLOCKED_NORMAL, L_ALL_BLOCKED, STOPPED, N, N}, - {N, F_TX_BLOCKED, N, N, T_TX_BLOCKED_PENDING, R_TX_BLOCKED_PENDING, F_TX_BLOCKED, N, F_ALL_BLOCKED, STOPPED, F_WORKING, N}, - {N, F_TX_BLOCKED, N, N, N, N, L_TX_BLOCKED, N, L_ALL_BLOCKED, STOPPED, N, N}, - {N, F_WORKING, N, N, N, N, L_TX_BLOCKED, L_BLOCKED_NORMAL, L_ALL_BLOCKED, STOPPED, N, L_WORKING}, -// - {N, F_TX_BLOCKED, L_TX_BLOCKED, T_TX_BLOCKED_PENDING, N, N, T_TX_BLOCKED_PENDING, N, T_ALL_BLOCKED_PENDING, STOPPED, N, N}, - {N, F_TX_BLOCKED, L_TX_BLOCKED, R_TX_BLOCKED_PENDING, N, N, R_TX_BLOCKED_PENDING, N, R_ALL_BLOCKED_PENDING, STOPPED, N, N}, -// - {N, F_ALL_BLOCKED, N, N, T_ALL_BLOCKED_PENDING, R_ALL_BLOCKED_PENDING, F_ALL_BLOCKED, N, F_ALL_BLOCKED, STOPPED, F_WORKING, N}, - {N, F_ALL_BLOCKED, L_ALL_BLOCKED, T_ALL_BLOCKED_PENDING, N, N, T_ALL_BLOCKED_PENDING,N, T_ALL_BLOCKED_PENDING, STOPPED, N, N}, - {N, F_ALL_BLOCKED, L_ALL_BLOCKED, R_ALL_BLOCKED_PENDING, N, N, R_ALL_BLOCKED_PENDING,N, R_ALL_BLOCKED_PENDING, STOPPED, N, N}, - {N, F_ALL_BLOCKED, N, N, N, N, L_ALL_BLOCKED, N, L_ALL_BLOCKED, STOPPED, N, N}, -// - {N, STOPPED, N, N, N, N, N, N, N, STOPPED, N, N}, - {N, N, N, N, N, N, N, N, N, N, N, N} - }; - - if (OB_UNLIKELY(!Ops::is_valid(op))) { - TRANS_LOG(WARN, "invalid argument", K(op)); - ret = OB_INVALID_ARGUMENT; - } else if (OB_UNLIKELY(!State::is_valid(state_))) { - TRANS_LOG(WARN, "ObLSTxCtxMgr current state is invalid", K_(state), K(op)); - ret = OB_ERR_UNEXPECTED; - } else { - const int64_t new_state = STATE_MAP[state_][op]; - if (OB_UNLIKELY(!State::is_valid(new_state))) { - ret = OB_STATE_NOT_MATCH; - } else { - last_state_ = state_; - state_ = new_state; - is_switching_ = true; - } - } - if (OB_SUCC(ret)) { - _TRANS_LOG(INFO, "ObLSTxCtxMgr switch state success(ls_id=%jd, %s ~> %s, op=%s)", - ls_id_.id(), State::state_str(last_state_), State::state_str(state_), Ops::op_str(op)); - } else { - _TRANS_LOG(WARN, "ObLSTxCtxMgr switch state failed(ret=%d, ls_id=%jd, state=%s, op=%s)", - ret, ls_id_.id(), State::state_str(state_), Ops::op_str(op)); - } - return ret; -} - -void ObLSTxCtxMgr::StateHelper::restore_state() -{ - if (is_switching_) { - is_switching_ = false; - state_ = last_state_; - } -} - int ObLSTxCtxMgr::create_tx_ctx(const ObTxCreateArg &arg, bool& existed, ObPartTransCtx *&ctx) @@ -422,7 +327,7 @@ int ObLSTxCtxMgr::create_tx_ctx_(const ObTxCreateArg &arg, } else if (!arg.for_replay_ && !is_master_()) { ret = OB_NOT_MASTER; } else if (!arg.for_replay_ && block) { - TRANS_LOG(WARN, "ObLSTxCtxMgr is blocked", K(arg), K(get_state_())); + TRANS_LOG(WARN, "ObLSTxCtxMgr is blocked", K(arg), K(tx_ls_state_mgr_)); ret = OB_PARTITION_IS_BLOCKED; } else if (is_stopped_()) { TRANS_LOG(WARN, "ObLSTxCtxMgr is stopped", K(arg)); @@ -675,62 +580,126 @@ int ObLSTxCtxMgr::replay_start_working_log(const ObTxStartWorkingLog &log, SCN s { int ret = OB_SUCCESS; UNUSED(log); - WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); - ReplayTxStartWorkingLogFunctor fn(start_working_ts); - if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { - TRANS_LOG(WARN, "[LsTxCtxMgr Role Change] replay start working log failed", KR(ret), K(ls_id_)); + + share::SCN tmp_applying_swl_scn; + if (OB_FAIL(retry_apply_start_working_log())) { + TRANS_LOG(WARN, "retry to apply start working log failed", K(ret), K(log), K(start_working_ts), + KPC(this)); + ret = OB_EAGAIN; + } else if (tx_ls_state_mgr_.need_retry_apply_SWL(tmp_applying_swl_scn)) + + { + if (tmp_applying_swl_scn < start_working_ts) { + ret = OB_EAGAIN; + TRANS_LOG(WARN, "need retry to apply prev start working log", K(ret), K(tmp_applying_swl_scn), + K(start_working_ts), K(log), KPC(this)); + } else { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(ERROR, "the applying SWL scn is larger than replaying SWL scn", K(ret), + K(tmp_applying_swl_scn), K(start_working_ts), K(log), KPC(this)); + } + } else { - TRANS_LOG(INFO, "[LsTxCtxMgr Role Change] replay start working log success", K(tenant_id_), K(ls_id_)); + + WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); + ReplayTxStartWorkingLogFunctor fn(start_working_ts); + if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { + FLOG_WARN("[LsTxCtxMgr Role Change] replay start working log failed", KR(ret), + K(ls_id_)); + } else { + tx_ls_state_mgr_.replay_SWL_succ(start_working_ts); + FLOG_INFO("[LsTxCtxMgr Role Change] replay start working log success", K(tenant_id_), + K(ls_id_)); + } } return ret; } +int ObLSTxCtxMgr::retry_apply_start_working_log() +{ + int ret = OB_SUCCESS; + + share::SCN retry_start_working_ts; + + if (tx_ls_state_mgr_.need_retry_apply_SWL(retry_start_working_ts)) { + ret = on_start_working_log_cb_succ(retry_start_working_ts); + } + + return ret; +} + int ObLSTxCtxMgr::on_start_working_log_cb_succ(SCN start_working_ts) { int ret = OB_SUCCESS; bool ignore_ret = false; WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); - StateHelper state_helper(ls_id_, state_); - if (is_t_pending_()) { - SwitchToLeaderFunctor fn(start_working_ts); - if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { - TRANS_LOG(WARN, "switch to leader failed", KR(ret), K(ls_id_)); - if (OB_NOT_MASTER == fn.get_ret()) { - // ignore ret - // PALF will switch to follower when submitting log return OB_NOT_MASTER - ignore_ret = true; + + share::SCN retry_start_working_ts; + if (!tx_ls_state_mgr_.waiting_SWL_cb()) { + TRANS_LOG(INFO, "This ls is not waiting start_working_cb. Skip the on_success operation", + K(ret), K(tx_ls_state_mgr_)); + } else if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::SWL_CB_SUCC, + start_working_ts))) { + TRANS_LOG(ERROR, "switch state fail", KR(ret), K(tenant_id_), K(ls_id_)); + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(errsim_apply_start_working_log())) { + TRANS_LOG(WARN, "errsim in on_start_working_log_cb_succ", K(ret), K(tenant_id_), K(ls_id_)); + } + + if (!tx_ls_state_mgr_.need_retry_apply_SWL(retry_start_working_ts)) { + TRANS_LOG(INFO, "This ls need not retry apply start_working. Skip the apply operation", K(ret), + K(tx_ls_state_mgr_)); + } else { + if (OB_FAIL(ret)) { + // do nothing + } else if (is_t_pending_()) { + SwitchToLeaderFunctor fn(start_working_ts); + if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { + TRANS_LOG(WARN, "switch to leader failed", KR(ret), K(ls_id_)); + if (OB_NOT_MASTER == fn.get_ret()) { + // ignore ret + // PALF will switch to follower when submitting log return OB_NOT_MASTER + ignore_ret = true; + } + } + } else if (is_r_pending_()) { + ResumeLeaderFunctor fn(start_working_ts); + if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { + TRANS_LOG(WARN, "resume leader failed", KR(ret), K(ls_id_)); + } + } else { + ret = OB_STATE_NOT_MATCH; + TRANS_LOG(ERROR, "unexpected state", KR(ret), K(ls_id_), K(tx_ls_state_mgr_)); + } + + if (OB_FAIL(ret)) { + if (ignore_ret) { + ret = OB_SUCCESS; + } + // TODO dingxi, takeover failed, notify palf to revoke itself + int tmp_ret = OB_SUCCESS; + // restore to follower + if (OB_TMP_FAIL(tx_ls_state_mgr_.switch_tx_ls_state( + ObTxLSStateMgr::TxLSAction::APPLY_SWL_FAIL, start_working_ts))) { + TRANS_LOG(ERROR, "restore follower failed", KR(tmp_ret), K(ls_id_), K(tx_ls_state_mgr_)); + ret = tmp_ret; + } + } else { + int tmp_ret = OB_SUCCESS; + if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::APPLY_SWL_SUCC, + start_working_ts))) { + TRANS_LOG(ERROR, "switch state failed", KR(ret), K(ls_id_), K(tx_ls_state_mgr_)); + } else { + leader_takeover_ts_ = MonotonicTs::current_time(); + try_wait_gts_and_inc_max_commit_ts_(); } } - } else if (is_r_pending_()) { - ResumeLeaderFunctor fn(start_working_ts); - if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { - TRANS_LOG(WARN, "resume leader failed", KR(ret), K(ls_id_)); - } - } else { - ret = OB_STATE_NOT_MATCH; - TRANS_LOG(ERROR, "unexpected state", KR(ret), K(ls_id_), K(state_)); } - if (OB_FAIL(ret)) { - if (ignore_ret) { - ret = OB_SUCCESS; - } - // TODO dingxi, takeover failed, notify palf to revoke itself - int tmp_ret = OB_SUCCESS; - // restore to follower - if (OB_TMP_FAIL(state_helper.switch_state(Ops::SWL_CB_FAIL))) { - TRANS_LOG(ERROR, "restore follower failed", KR(tmp_ret), K(ls_id_), K(state_)); - ret = tmp_ret; - } - } else { - int tmp_ret = OB_SUCCESS; - if (OB_FAIL(state_helper.switch_state(Ops::SWL_CB_SUCC))) { - TRANS_LOG(ERROR, "switch state failed", KR(ret), K(ls_id_), K(state_)); - } else { - leader_takeover_ts_ = MonotonicTs::current_time(); - try_wait_gts_and_inc_max_commit_ts_(); - } - } - TRANS_LOG(INFO, "[LsTxCtxMgr Role Change] on_start_working_log_cb_succ", K(ret), KPC(this)); + FLOG_INFO("[LsTxCtxMgr Role Change] on_start_working_log_cb_succ", K(ret), K(start_working_ts), + KPC(this)); return ret; } @@ -738,11 +707,10 @@ int ObLSTxCtxMgr::on_start_working_log_cb_fail() { int ret = OB_SUCCESS; WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); - StateHelper state_helper(ls_id_, state_); - if (OB_FAIL(state_helper.switch_state(Ops::SWL_CB_FAIL))) { + if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::SWL_CB_FAIL))) { TRANS_LOG(WARN, "switch state fail", KR(ret), K(tenant_id_), K(ls_id_)); } - TRANS_LOG(INFO, "[LsTxCtxMgr Role Change] on_start_working_log_cb_fail", K(ret), KPC(this)); + FLOG_INFO("[LsTxCtxMgr Role Change] on_start_working_log_cb_fail", K(ret), KPC(this)); return ret; } @@ -751,7 +719,9 @@ int ObLSTxCtxMgr::submit_start_working_log_() int ret = OB_SUCCESS; SCN scn; const int64_t fake_epoch = 0xbaba; - if (OB_FAIL(ls_log_writer_.submit_start_working_log(fake_epoch, scn))) { + if (OB_FAIL(errsim_submit_start_working_log())) { + TRANS_LOG(WARN, "errsim in submit start working log failed", K(ret), KPC(this)); + } else if (OB_FAIL(ls_log_writer_.submit_start_working_log(fake_epoch, scn))) { TRANS_LOG(WARN, "submit start working log failed", KR(ret), K(*this)); } return ret; @@ -764,13 +734,12 @@ int ObLSTxCtxMgr::switch_to_follower_forcedly() ObTxCommitCallback *cb_list = NULL; { WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); - StateHelper state_helper(ls_id_, state_); if (IS_NOT_INIT) { TRANS_LOG(ERROR, "ObLSTxCtxMgr not inited", K(ls_id_)); ret = OB_NOT_INIT; } else if (is_follower_()) { // already follower, do nothing - } else if (OB_FAIL(state_helper.switch_state(Ops::LEADER_REVOKE))) { + } else if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::LEADER_REVOKE_FORCEDLLY))) { TRANS_LOG(ERROR, "switch state error", KR(ret), "manager", *this); } else { SwitchToFollowerForcedlyFunctor fn(cb_list); @@ -781,7 +750,7 @@ int ObLSTxCtxMgr::switch_to_follower_forcedly() } if (OB_FAIL(ret)) { - state_helper.restore_state(); + tx_ls_state_mgr_.restore_tx_ls_state(); } } } @@ -791,7 +760,7 @@ int ObLSTxCtxMgr::switch_to_follower_forcedly() if (timeguard.get_diff() > 3 * 1000000) { TRANS_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "switch_to_follower_forcedly use too much time", K(timeguard), "manager", *this); } - TRANS_LOG(INFO, "[LsTxCtxMgr Role Change] switch_to_follower_forcedly", K(ret), KPC(this)); + FLOG_INFO("[LsTxCtxMgr Role Change] switch_to_follower_forcedly", K(ret), KPC(this)); return ret; } @@ -833,22 +802,27 @@ int ObLSTxCtxMgr::try_wait_gts_and_inc_max_commit_ts_() int ObLSTxCtxMgr::switch_to_leader() { int ret = OB_SUCCESS; - WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); - StateHelper state_helper(ls_id_, state_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K(ls_id_)); - } else if (OB_FAIL(state_helper.switch_state(Ops::LEADER_TAKEOVER))) { - TRANS_LOG(WARN, "switch state failed", KR(ret), K(ls_id_)); + + if (OB_FAIL(retry_apply_start_working_log())) { + TRANS_LOG(WARN, "retry to apply prev start working log failed", K(ret), KPC(this)); } else { - if (OB_FAIL(submit_start_working_log_())) { - TRANS_LOG(WARN, "submit start working log failed", KR(ret), K(ls_id_)); - } - if (OB_FAIL(ret)) { - state_helper.restore_state(); + WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "not init", KR(ret), K(ls_id_)); + } else if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state( + ObTxLSStateMgr::TxLSAction::LEADER_TAKEOVER))) { + TRANS_LOG(WARN, "switch state failed", KR(ret), K(ls_id_)); + } else { + if (OB_FAIL(submit_start_working_log_())) { + TRANS_LOG(WARN, "submit start working log failed", KR(ret), K(ls_id_)); + } + if (OB_FAIL(ret)) { + tx_ls_state_mgr_.restore_tx_ls_state(); + } } } - TRANS_LOG(INFO, "[LsTxCtxMgr] switch_to_leader", K(ret), KPC(this)); + FLOG_INFO("[LsTxCtxMgr Role Change] switch_to_leader", K(ret), KPC(this)); return ret; } @@ -856,13 +830,16 @@ int ObLSTxCtxMgr::switch_to_follower_gracefully() { int ret = OB_SUCCESS; ObTimeGuard timeguard("switch_to_follower_gracefully"); - StateHelper state_helper(ls_id_, state_); int64_t start_time = ObTimeUtility::current_time(); int64_t process_count = 0; while (OB_SUCC(ret) && is_pending_()) { if (ObTimeUtility::current_time() - start_time >= WAIT_SW_CB_TIMEOUT) { ret = OB_TIMEOUT; - TRANS_LOG(WARN, "start working cb waiting timeout", K(ret)); + TRANS_LOG(WARN, "start working cb waiting timeout", K(ret), KPC(this)); + if (tx_ls_state_mgr_.is_start_working_apply_pending()) { + ret = OB_LS_NEED_REVOKE; + TRANS_LOG(WARN, "apply start working log failed with waiting timeout, need revoke", K(ret), KPC(this)); + } } else { ob_usleep(WAIT_SW_CB_INTERVAL); } @@ -870,17 +847,18 @@ int ObLSTxCtxMgr::switch_to_follower_gracefully() timeguard.click(); ObTxCommitCallback *cb_list = NULL; - { + if (OB_SUCC(ret)) { WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); timeguard.click(); if (OB_FAIL(ret)) { - //do nothing + // do nothing } else if (IS_NOT_INIT) { ret = OB_NOT_INIT; TRANS_LOG(WARN, "not init", KR(ret), K(ls_id_)); - } else if (OB_FAIL(state_helper.switch_state(Ops::LEADER_REVOKE))) { - TRANS_LOG(WARN, "switch state error", KR(ret), K(tenant_id_), K(ls_id_), K(state_)); + } else if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state( + ObTxLSStateMgr::TxLSAction::LEADER_REVOKE_GRACEFULLY))) { + TRANS_LOG(WARN, "switch state error", KR(ret), K(tenant_id_), K(ls_id_), K(tx_ls_state_mgr_)); } else { timeguard.click(); // TODO @@ -889,13 +867,17 @@ int ObLSTxCtxMgr::switch_to_follower_gracefully() if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { TRANS_LOG(WARN, "for each tx ctx error", KR(ret), "manager", *this); ret = fn.get_ret(); + } else if (OB_FAIL(errsim_switch_to_followr_gracefully())) { + TRANS_LOG(WARN, "errsim in switch_to_follower_gracefully", K(ret), K(tenant_id_), + K(ls_id_)); } process_count = fn.get_count(); timeguard.click(); if (OB_FAIL(ret)) { int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(state_helper.switch_state(Ops::RESUME_LEADER))) { - TRANS_LOG(WARN, "switch state error", KR(ret), K(ls_id_), K(state_)); + if (OB_TMP_FAIL( + tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::RESUME_LEADER))) { + TRANS_LOG(WARN, "switch state error", KR(ret), K(ls_id_), K(tx_ls_state_mgr_)); } else if (OB_TMP_FAIL(submit_start_working_log_())) { TRANS_LOG(WARN, "submit start working log failed", KR(tmp_ret), K(*this)); } @@ -912,7 +894,8 @@ int ObLSTxCtxMgr::switch_to_follower_gracefully() } (void)process_callback_(cb_list); timeguard.click(); - TRANS_LOG(INFO, "[LsTxCtxMgr] switch_to_follower_gracefully", K(ret), KPC(this), K(process_count)); + FLOG_INFO("[LsTxCtxMgr Role Change] switch_to_follower_gracefully", K(ret), KPC(this), + K(process_count)); if (timeguard.get_diff() > 1000000) { TRANS_LOG(WARN, "use too much time", K(timeguard), K(process_count)); } @@ -923,27 +906,31 @@ int ObLSTxCtxMgr::switch_to_follower_gracefully() int ObLSTxCtxMgr::resume_leader() { int ret = OB_SUCCESS; - WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); - StateHelper state_helper(ls_id_, state_); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not init", KR(ret), K(ls_id_)); - } else if (OB_FAIL(state_helper.switch_state(Ops::RESUME_LEADER))) { - TRANS_LOG(WARN, "switch state error", KR(ret), K(tenant_id_), K(ls_id_), K(state_)); + if (OB_FAIL(retry_apply_start_working_log())) { + TRANS_LOG(WARN, "retry to apply prev start working log failed", K(ret), KPC(this)); } else { - // TODO dingxi, previous logs need to be applied done before submit start working log - if (OB_FAIL(submit_start_working_log_())) { - TRANS_LOG(WARN, "submit start working log failed", KR(ret), K(*this)); - } - if (OB_FAIL(ret)) { - state_helper.restore_state(); - TRANS_LOG(WARN, "resume leader failed", KR(ret), K(*this)); + WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "not init", KR(ret), K(ls_id_)); + } else if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state( + ObTxLSStateMgr::TxLSAction::RESUME_LEADER))) { + TRANS_LOG(WARN, "switch state error", KR(ret), K(tenant_id_), K(ls_id_), K(tx_ls_state_mgr_)); } else { - is_leader_serving_ = true; - TRANS_LOG(INFO, "resume leader success", K(*this)); + // previous active info logs will be filter by start_working_ts in part_ctx + if (OB_FAIL(submit_start_working_log_())) { + TRANS_LOG(WARN, "submit start working log failed", KR(ret), K(*this)); + } + if (OB_FAIL(ret)) { + tx_ls_state_mgr_.restore_tx_ls_state(); + TRANS_LOG(WARN, "resume leader failed", KR(ret), K(*this)); + } else { + is_leader_serving_ = true; + TRANS_LOG(INFO, "resume leader success", K(*this)); + } } } - TRANS_LOG(INFO, "[LsTxCtxMgr] resume_leader", K(ret), KPC(this)); + TRANS_LOG(INFO, "[LsTxCtxMgr Role Change] resume_leader", K(ret), KPC(this)); return ret; } @@ -968,7 +955,6 @@ bool ObLSTxCtxMgr::in_leader_serving_state() int ObLSTxCtxMgr::stop(const bool graceful) { int ret = OB_SUCCESS; - StateHelper state_helper(ls_id_, state_); ObTxCommitCallback *cb_list = NULL; const KillTransArg arg(graceful); ObTimeGuard timeguard("ctxmgr stop"); @@ -983,7 +969,7 @@ int ObLSTxCtxMgr::stop(const bool graceful) } else { { WLockGuard guard(minor_merge_lock_); - if (OB_FAIL(state_helper.switch_state(Ops::STOP))) { + if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::STOP))) { TRANS_LOG(WARN, "switch state error", KR(ret), "manager", *this); } } @@ -997,7 +983,7 @@ int ObLSTxCtxMgr::stop(const bool graceful) TRANS_LOG(WARN, "for each transaction context error", KR(ret), "manager", *this); } if (OB_FAIL(ret)) { - state_helper.restore_state(); + tx_ls_state_mgr_.restore_tx_ls_state(); } } } @@ -1038,12 +1024,11 @@ int ObLSTxCtxMgr::kill_all_tx(const bool graceful, bool &is_all_tx_cleaned_up) int ObLSTxCtxMgr::block_tx(bool &is_all_tx_cleaned_up) { int ret = OB_SUCCESS; - StateHelper state_helper(ls_id_, state_); WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); if (is_stopped_()) { TRANS_LOG(WARN, "ls_tx_ctx_mgr is stopped, not need block"); - } else if (OB_FAIL(state_helper.switch_state(Ops::BLOCK_TX))) { + } else if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::BLOCK_START_TX))) { TRANS_LOG(WARN, "switch state error", KR(ret), "manager", *this); } else { is_all_tx_cleaned_up = (get_tx_ctx_count() == 0); @@ -1055,12 +1040,11 @@ int ObLSTxCtxMgr::block_tx(bool &is_all_tx_cleaned_up) int ObLSTxCtxMgr::block_all(bool &is_all_tx_cleaned_up) { int ret = OB_SUCCESS; - StateHelper state_helper(ls_id_, state_); WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); if (is_stopped_()) { TRANS_LOG(WARN, "ls_tx_ctx_mgr is stopped, not need block"); - } else if (OB_FAIL(state_helper.switch_state(Ops::BLOCK_ALL))) { + } else if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::BLOCK_START_WR))) { TRANS_LOG(WARN, "switch state error", KR(ret), "manager", *this); } else { is_all_tx_cleaned_up = (get_tx_ctx_count() == 0); @@ -1072,10 +1056,9 @@ int ObLSTxCtxMgr::block_all(bool &is_all_tx_cleaned_up) int ObLSTxCtxMgr::block_normal(bool &is_all_tx_cleaned_up) { int ret = OB_SUCCESS; - StateHelper state_helper(ls_id_, state_); WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); - if (OB_FAIL(state_helper.switch_state(Ops::BLOCK_NORMAL))) { + if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::BLOCK_START_NORMAL_TX))) { TRANS_LOG(WARN, "switch state error", KR(ret), "manager", *this); } else { is_all_tx_cleaned_up = (get_tx_ctx_count() == 0); @@ -1087,10 +1070,9 @@ int ObLSTxCtxMgr::block_normal(bool &is_all_tx_cleaned_up) int ObLSTxCtxMgr::online() { int ret = OB_SUCCESS; - StateHelper state_helper(ls_id_, state_); WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); - if (OB_FAIL(state_helper.switch_state(Ops::ONLINE))) { + if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::ONLINE))) { TRANS_LOG(WARN, "switch state error", KR(ret), "manager", *this); } else { online_ts_ = ObTimeUtility::current_time(); @@ -1102,10 +1084,9 @@ int ObLSTxCtxMgr::online() int ObLSTxCtxMgr::unblock_normal() { int ret = OB_SUCCESS; - StateHelper state_helper(ls_id_, state_); WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); - if (OB_FAIL(state_helper.switch_state(Ops::UNBLOCK_NORMAL))) { + if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::UNBLOCK_NORMAL_TX))) { TRANS_LOG(WARN, "switch state error", KR(ret), "manager", *this); } TRANS_LOG(INFO, "unblock ls normally", K(ret), "manager", *this); @@ -2515,34 +2496,110 @@ int ObLSTxCtxMgr::do_standby_cleanup() return ret; } -int ObLSTxCtxMgr::transfer_out_tx_op(int64_t except_tx_id, - const SCN data_end_scn, - const SCN op_scn, - NotifyType op_type, - bool is_replay, - ObLSID dest_ls_id, - int64_t transfer_epoch, +OB_NOINLINE int ObLSTxCtxMgr::errsim_switch_to_followr_gracefully() +{ + int ret = OB_SUCCESS; + +#ifdef ERRSIM + ret = EN_SWITCH_TO_FOLLOWER_GRACEFULLY; +#endif + + if (OB_FAIL(ret)) { + TRANS_LOG(WARN, "errsim for switch_to_follower_gracefully", K(ret), KPC(this)); + } + + return ret; +} + +OB_NOINLINE int ObLSTxCtxMgr::errsim_submit_start_working_log() +{ + int ret = OB_SUCCESS; + +#ifdef ERRSIM + ret = EN_SUBMIT_START_WORKING_LOG; +#endif + + if (OB_FAIL(ret)) { + TRANS_LOG(WARN, "errsim for submit_start_working_log", K(ret), KPC(this)); + } + + return ret; +} + +OB_NOINLINE int ObLSTxCtxMgr::errsim_apply_start_working_log() +{ + int ret = OB_SUCCESS; + +#ifdef ERRSIM + ret = EN_APPLY_START_WORKING_LOG; +#endif + + if (OB_FAIL(ret)) { + TRANS_LOG(WARN, "errsim for apply_start_working_log", K(ret), KPC(this)); + } + + return ret; +} + +int ObLSTxCtxMgr::filter_tx_need_transfer(ObIArray &tablet_list, + const share::SCN data_end_scn, + ObIArray &move_tx_ids) +{ + int ret = OB_SUCCESS; + FilterTransferTxFunctor fn(tablet_list, data_end_scn, move_tx_ids); + if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { + ret = fn.get_ret(); + TRANS_LOG(WARN, "for each tx ctx error", KR(ret), "manager", *this); + } + return ret; +} + +int ObLSTxCtxMgr::transfer_out_tx_op(const ObTransferOutTxParam ¶m, int64_t& active_tx_count, int64_t &op_tx_count) { int ret = OB_SUCCESS; - const int64_t abs_expired_time = INT64_MAX; - TransferOutTxOpFunctor fn(abs_expired_time, except_tx_id, - data_end_scn, - op_scn, - op_type, - is_replay, - dest_ls_id, - transfer_epoch); - if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { - TRANS_LOG(WARN, "for each tx ctx error", KR(ret), "manager", *this); - ret = fn.get_ret(); + if (OB_ISNULL(param.move_tx_ids_)) { + TransferOutTxOpFunctor fn(param); + if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { + ret = fn.get_ret(); + TRANS_LOG(WARN, "for each tx ctx error", KR(ret), "manager", *this); + } else { + active_tx_count = fn.get_count(); + op_tx_count = fn.get_op_tx_count(); + } } else { - active_tx_count = fn.get_count(); - op_tx_count = fn.get_op_tx_count(); + active_tx_count = ls_tx_ctx_map_.count(); + for (int64_t idx = 0; OB_SUCC(ret) && idx < param.move_tx_ids_->count(); idx++) { + if (param.move_tx_ids_->at(idx).get_id() == param.except_tx_id_) { + continue; + } + ObPartTransCtx *ctx = nullptr; + ObTransCtx *tmp_ctx = nullptr; + bool is_operated = false; + if (OB_FAIL(ls_tx_ctx_map_.get(param.move_tx_ids_->at(idx), tmp_ctx))) { + if (OB_ENTRY_NOT_EXIST != ret) { + TRANS_LOG(WARN, "get tx ctx failed", KR(ret), K(param.move_tx_ids_->at(idx))); + } else { + ret = OB_SUCCESS; + } + } else if (FALSE_IT(ctx = static_cast(tmp_ctx))) { + } else if (OB_FAIL(ctx->do_transfer_out_tx_op(param.data_end_scn_, + param.op_scn_, + param.op_type_, + param.is_replay_, + param.dest_ls_id_, + param.transfer_epoch_, + is_operated))) { + TRANS_LOG(WARN, "transfer out tx failed", KR(ret), K(param)); + } else if (is_operated) { + op_tx_count++; + } + if (OB_NOT_NULL(ctx)) { + revert_tx_ctx(ctx); + } + } } - TRANS_LOG(INFO, "[TRANSFER] transfer_out_tx_op", KR(ret), K(data_end_scn), K(op_scn), K(op_type), K(is_replay), K(dest_ls_id), - K(transfer_epoch), K(active_tx_count), K(op_tx_count), K(ls_tx_ctx_map_.count()), K(tenant_id_), K(ls_id_)); return ret; } @@ -2568,23 +2625,43 @@ int ObLSTxCtxMgr::wait_tx_write_end(ObTimeoutCtx &timeout_ctx) int ObLSTxCtxMgr::collect_tx_ctx(const ObLSID dest_ls_id, const SCN log_scn, const ObIArray &tablet_list, - int64_t &tx_count, + const ObIArray &move_tx_ids, int64_t &collect_count, ObIArray &res) { int ret = OB_SUCCESS; - - const int64_t abs_expired_time = INT64_MAX; - CollectTxCtxFunctor fn(abs_expired_time, dest_ls_id, log_scn, tablet_list, tx_count, collect_count, res); - if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { - TRANS_LOG(WARN, "for each tx ctx error", KR(ret), "manager", *this); - ret = fn.get_ret(); - } else { - tx_count = fn.get_tx_count(); - collect_count = fn.get_collect_count(); + ObSEArray final_move_tx_ids; + for (int64_t idx = 0; OB_SUCC(ret) && idx < move_tx_ids.count(); idx++) { + ObPartTransCtx *ctx = nullptr; + ObTransCtx *tmp_ctx = nullptr; + bool is_collected = false; + ObTxCtxMoveArg arg; + if (OB_FAIL(ls_tx_ctx_map_.get(move_tx_ids.at(idx), tmp_ctx))) { + if (OB_ENTRY_NOT_EXIST != ret) { + TRANS_LOG(WARN, "get tx ctx failed", KR(ret), K(move_tx_ids.at(idx))); + } else { + ret = OB_SUCCESS; + } + } else if (FALSE_IT(ctx = static_cast(tmp_ctx))) { + } else if (OB_FAIL(ctx->collect_tx_ctx(dest_ls_id, + log_scn, + tablet_list, + arg, + is_collected))) { + TRANS_LOG(WARN, "collect tx ctx failed", KR(ret), K(move_tx_ids.at(idx))); + } else if (!is_collected) { + } else if (OB_FAIL(final_move_tx_ids.push_back(move_tx_ids.at(idx)))) { + TRANS_LOG(WARN, "collect tx ctx failed", KR(ret), K(move_tx_ids.at(idx))); + } else if (OB_FAIL(res.push_back(arg))) { + TRANS_LOG(WARN, "push to array failed", KR(ret), K(move_tx_ids.at(idx))); + } else { + collect_count++; + } + if (OB_NOT_NULL(ctx)) { + revert_tx_ctx(ctx); + } } - - TRANS_LOG(INFO, "collect_tx_ctx", KR(ret), K(tx_count), K(collect_count), K(tenant_id_), K(ls_id_)); + TRANS_LOG(INFO, "collect_tx_ctx", KR(ret), K(final_move_tx_ids), K(collect_count), K(tenant_id_), K(ls_id_)); return ret; } @@ -2694,11 +2771,10 @@ int ObLSTxCtxMgr::move_tx_op(const ObTransferMoveTxParam &move_tx_param, if (OB_NOT_NULL(ctx)) { revert_tx_ctx(ctx); } - TRANS_LOG(INFO, "move_tx_op", KR(ret), K(arg.tx_id_), K(ls_id_), K(is_replay), K(is_created)); + TRANS_LOG(INFO, "move_tx_op", KR(ret), K(arg.tx_id_), K(ls_id_), K(is_replay), K(is_created), K(move_tx_param.op_type_)); } return ret; } - } } diff --git a/src/storage/tx/ob_trans_ctx_mgr_v4.h b/src/storage/tx/ob_trans_ctx_mgr_v4.h index 908f1c1b61..282ef9e68e 100644 --- a/src/storage/tx/ob_trans_ctx_mgr_v4.h +++ b/src/storage/tx/ob_trans_ctx_mgr_v4.h @@ -19,6 +19,7 @@ #include "common/ob_simple_iterator.h" #include "storage/tx/ob_trans_ctx.h" #include "storage/tx/ob_tx_ls_log_writer.h" +#include "storage/tx/ob_tx_ls_state_mgr.h" #include "storage/tx/ob_tx_retain_ctx_mgr.h" #include "storage/tablelock/ob_lock_table.h" #include "storage/tx/ob_keep_alive_ls_handler.h" @@ -40,6 +41,7 @@ class ObTransSubmitLogFunctor; class ObTxCtxTable; struct ObTxCtxMoveArg; struct ObTransferMoveTxParam; +struct ObTransferOutTxParam; } namespace memtable @@ -202,20 +204,18 @@ public: // Offline the in-memory state of the ObLSTxCtxMgr int offline(); - int transfer_out_tx_op(int64_t except_tx_id, - const SCN data_end_scn, - const SCN op_scn, - NotifyType op_type, - bool is_replay, - ObLSID dest_ls_id, - int64_t transfer_epoch, + int filter_tx_need_transfer(ObIArray &tablet_list, + const share::SCN data_end_scn, + ObIArray &move_tx_ids); + + int transfer_out_tx_op(const ObTransferOutTxParam ¶m, int64_t& active_tx_count, int64_t &op_tx_count); int wait_tx_write_end(ObTimeoutCtx &timeout_ctx); int collect_tx_ctx(const share::ObLSID dest_ls_id, const SCN log_scn, const ObIArray &tablet_list, - int64_t &tx_count, + const ObIArray &move_tx_ids, int64_t &colllect_count, ObIArray &res); int move_tx_op(const ObTransferMoveTxParam &move_tx_param, @@ -515,6 +515,7 @@ public: // ObLSTxCtxMgr's status, drive ObLSTxCtxMgr to continue to execute the switch_to_leader routine // or resume_leader routine; int on_start_working_log_cb_succ(share::SCN start_working_scn); + int retry_apply_start_working_log(); // START_WORKING log entry failed to written to the PALF; // Break the switch_to_leader routine or resume_leader routine; switch the ObLSTxCtxMgr's state to F_WORKING @@ -550,9 +551,6 @@ public: // Get the tenant_id corresponding to this ObLSTxCtxMgr; int64_t get_tenant_id() { return tenant_id_; } - // Get the state of this ObLSTxCtxMgr - int64_t get_state() { return get_state_(); } - // check is master bool is_master() const { return is_master_(); } @@ -573,11 +571,14 @@ public: int do_standby_cleanup(); + int errsim_switch_to_followr_gracefully(); + int errsim_submit_start_working_log(); + int errsim_apply_start_working_log(); + TO_STRING_KV(KP(this), K_(ls_id), K_(tenant_id), - "state", - State::state_str(state_), + K_(tx_ls_state_mgr), K_(total_tx_ctx_count), K_(active_tx_count), K_(ls_retain_ctx_mgr), @@ -611,219 +612,42 @@ public: static const int64_t WAIT_SW_CB_INTERVAL = 10 * 1000; // 10 ms static const int64_t WAIT_READONLY_REQUEST_TIME = 10 * 1000 * 1000; static const int64_t READONLY_REQUEST_TRACE_ID_NUM = 8192; -private: - class State - { - public: - static const int64_t INVALID = -1; - static const int64_t INIT = 0; - static const int64_t F_WORKING = 1; - static const int64_t T_PENDING = 2; - static const int64_t R_PENDING = 3; - static const int64_t L_WORKING = 4; - static const int64_t F_TX_BLOCKED = 5; - static const int64_t L_TX_BLOCKED = 6; - static const int64_t L_BLOCKED_NORMAL = 7; - static const int64_t T_TX_BLOCKED_PENDING = 8; - static const int64_t R_TX_BLOCKED_PENDING = 9; - static const int64_t F_ALL_BLOCKED = 10; - static const int64_t T_ALL_BLOCKED_PENDING = 11; - static const int64_t R_ALL_BLOCKED_PENDING = 12; - static const int64_t L_ALL_BLOCKED = 13; - static const int64_t STOPPED = 14; - static const int64_t END = 15; - static const int64_t MAX = 16; - public: - static bool is_valid(const int64_t state) - { return state > INVALID && state < MAX; } - - #define TCM_STATE_CASE_TO_STR(state) \ - case state: \ - str = #state; \ - break; - - static const char* state_str(uint64_t state) - { - const char* str = "INVALID"; - switch (state) { - TCM_STATE_CASE_TO_STR(INIT); - TCM_STATE_CASE_TO_STR(F_WORKING); - TCM_STATE_CASE_TO_STR(T_PENDING); - TCM_STATE_CASE_TO_STR(R_PENDING); - TCM_STATE_CASE_TO_STR(L_WORKING); - TCM_STATE_CASE_TO_STR(F_TX_BLOCKED); - TCM_STATE_CASE_TO_STR(L_TX_BLOCKED); - TCM_STATE_CASE_TO_STR(L_BLOCKED_NORMAL); - TCM_STATE_CASE_TO_STR(T_TX_BLOCKED_PENDING); - TCM_STATE_CASE_TO_STR(R_TX_BLOCKED_PENDING); - - TCM_STATE_CASE_TO_STR(F_ALL_BLOCKED); - TCM_STATE_CASE_TO_STR(T_ALL_BLOCKED_PENDING); - TCM_STATE_CASE_TO_STR(R_ALL_BLOCKED_PENDING); - TCM_STATE_CASE_TO_STR(L_ALL_BLOCKED); - - TCM_STATE_CASE_TO_STR(STOPPED); - TCM_STATE_CASE_TO_STR(END); - default: - break; - } - return str; - } - #undef TCM_STATE_CASE_TO_STR - }; - - class Ops - { - public: - static const int64_t INVALID = -1; - static const int64_t START = 0; - static const int64_t LEADER_REVOKE = 1; - static const int64_t SWL_CB_SUCC = 2;// start working log callback success - static const int64_t SWL_CB_FAIL = 3;// start working log callback failed - static const int64_t LEADER_TAKEOVER = 4; - static const int64_t RESUME_LEADER = 5; - static const int64_t BLOCK_TX = 6; - static const int64_t BLOCK_NORMAL = 7; - static const int64_t BLOCK_ALL = 8; - static const int64_t STOP = 9; - static const int64_t ONLINE = 10; - static const int64_t UNBLOCK_NORMAL = 11; - static const int64_t MAX = 12; - public: - static bool is_valid(const int64_t op) - { return op > INVALID && op < MAX; } - - #define TCM_OP_CASE_TO_STR(op) \ - case op: \ - str = #op; \ - break; - - static const char* op_str(uint64_t op) - { - const char* str = "INVALID"; - switch (op) { - TCM_OP_CASE_TO_STR(START); - TCM_OP_CASE_TO_STR(LEADER_REVOKE); - TCM_OP_CASE_TO_STR(SWL_CB_SUCC); - TCM_OP_CASE_TO_STR(SWL_CB_FAIL); - TCM_OP_CASE_TO_STR(LEADER_TAKEOVER); - TCM_OP_CASE_TO_STR(RESUME_LEADER); - TCM_OP_CASE_TO_STR(BLOCK_TX); - TCM_OP_CASE_TO_STR(BLOCK_NORMAL); - TCM_OP_CASE_TO_STR(BLOCK_ALL); - TCM_OP_CASE_TO_STR(STOP); - TCM_OP_CASE_TO_STR(ONLINE); - TCM_OP_CASE_TO_STR(UNBLOCK_NORMAL); - default: - break; - } - return str; - } - #undef TCM_OP_CASE_TO_STR - }; - - class StateHelper - { - public: - explicit StateHelper(const share::ObLSID &ls_id, int64_t &state) - : ls_id_(ls_id), state_(state), last_state_(State::INVALID), is_switching_(false) {} - ~StateHelper() {} - - int switch_state(const int64_t op); - void restore_state(); - int64_t get_state() const { return state_; } - private: - const share::ObLSID &ls_id_; - int64_t &state_; - int64_t last_state_; - bool is_switching_; - }; private: inline bool is_master_() const - { return is_master_(ATOMIC_LOAD(&state_)); } - inline bool is_master_(int64_t state) const - { return State::L_WORKING == state || - State::L_TX_BLOCKED == state || - State::L_BLOCKED_NORMAL == state || - State::L_ALL_BLOCKED == state; } + { return tx_ls_state_mgr_.is_master(); } inline bool is_follower_() const - { return is_follower_(ATOMIC_LOAD(&state_)); } - inline bool is_follower_(int64_t state) const - { return State::F_WORKING == state || - State::F_TX_BLOCKED == state || - State::F_ALL_BLOCKED == state; } + { return tx_ls_state_mgr_.is_follower(); } inline bool is_tx_blocked_() const - { return is_tx_blocked_(ATOMIC_LOAD(&state_)); } - inline bool is_tx_blocked_(int64_t state) const - { - return State::F_TX_BLOCKED == state || - State::L_TX_BLOCKED == state || - State::T_TX_BLOCKED_PENDING == state || - State::R_TX_BLOCKED_PENDING == state || - State::F_ALL_BLOCKED == state || - State::T_ALL_BLOCKED_PENDING == state || - State::R_ALL_BLOCKED_PENDING == state || - State::L_ALL_BLOCKED == state; - } + { return tx_ls_state_mgr_.is_block_start_tx(); } inline bool is_normal_blocked_() const - { return is_normal_blocked_(ATOMIC_LOAD(&state_)); } - inline bool is_normal_blocked_(int64_t state) const - { return State::L_BLOCKED_NORMAL == state; } + { return tx_ls_state_mgr_.is_block_start_normal_tx(); } inline bool is_all_blocked_() const - { return is_all_blocked_(ATOMIC_LOAD(&state_)); } - inline bool is_all_blocked_(const int64_t state) const - { - return State::F_ALL_BLOCKED == state || - State::T_ALL_BLOCKED_PENDING == state || - State::R_ALL_BLOCKED_PENDING == state || - State::L_ALL_BLOCKED == state; - } + { return tx_ls_state_mgr_.is_block_WR(); } // check pending substate inline bool is_t_pending_() const - { return is_t_pending_(ATOMIC_LOAD(&state_)); } - inline bool is_t_pending_(int64_t state) const - { - return State::T_PENDING == state || - State::T_TX_BLOCKED_PENDING == state || - State::T_ALL_BLOCKED_PENDING == state; - } + { return tx_ls_state_mgr_.is_switch_leader_pending(); } inline bool is_r_pending_() const - { return is_r_pending_(ATOMIC_LOAD(&state_)); } - inline bool is_r_pending_(int64_t state) const - { - return State::R_PENDING == state || - State::R_TX_BLOCKED_PENDING == state || - State::R_ALL_BLOCKED_PENDING == state; - } + { return tx_ls_state_mgr_.is_resume_leader_pending(); } inline bool is_pending_() const - { return is_pending_(ATOMIC_LOAD(&state_)); } - inline bool is_pending_(int64_t state) const - { - return is_t_pending_(state) || is_r_pending_(state); - } + { return tx_ls_state_mgr_.is_leader_takeover_pending(); } inline bool is_stopped_() const - { return is_stopped_(ATOMIC_LOAD(&state_)); } - inline bool is_stopped_(int64_t state) const - { return State::STOPPED == state; } - - int64_t get_state_() const - { return ATOMIC_LOAD(&state_); } + { return tx_ls_state_mgr_.is_stopped(); } private: // Identifies this ObLSTxCtxMgr is inited or not; bool is_inited_; // See the ObLSTxCtxMgr's internal class State - int64_t state_; + ObTxLSStateMgr tx_ls_state_mgr_; // A thread-safe hashmap, used to find and traverse TxCtx in this ObLSTxCtxMgr ObLSTxCtxMap ls_tx_ctx_map_; diff --git a/src/storage/tx/ob_trans_define_v4.cpp b/src/storage/tx/ob_trans_define_v4.cpp index aebf84efe6..db6b11d5d4 100644 --- a/src/storage/tx/ob_trans_define_v4.cpp +++ b/src/storage/tx/ob_trans_define_v4.cpp @@ -183,7 +183,9 @@ DEF_TO_STRING(ObTxSavePoint) J_OBJ_END(); return pos; } -OB_SERIALIZE_MEMBER(ObTxExecResult, incomplete_, parts_, conflict_txs_, conflict_info_array_); +OB_SERIALIZE_MEMBER(ObTxExecResult, incomplete_, parts_, + conflict_txs_,// FARM COMPAT WHITELIST for cflict_txs_ + conflict_info_array_); OB_SERIALIZE_MEMBER(ObTxSnapshot, tx_id_, version_, scn_, elr_); OB_SERIALIZE_MEMBER(ObTxReadSnapshot, valid_, diff --git a/src/storage/tx/ob_trans_define_v4.h b/src/storage/tx/ob_trans_define_v4.h index a0a1d7d410..27ac0ec2c8 100644 --- a/src/storage/tx/ob_trans_define_v4.h +++ b/src/storage/tx/ob_trans_define_v4.h @@ -352,7 +352,7 @@ class ObTxExecResult bool incomplete_; // TODO: (yunxing.cyx) remove, required before sql use new API share::ObLSArray touched_ls_list_; ObTxPartList parts_; - ObSArray conflict_txs_; // FARM COMPAT WHITELIST + ObSArray conflict_txs_; // FARM COMPAT WHITELIST for cflict_txs_ ObSArray conflict_info_array_; public: ObTxExecResult(); @@ -554,7 +554,7 @@ protected: // conflict_txs_ is valid when transaction is not executed on local // on scheduler, conflict_txs_ merges all participants executed results on remote // on participant, conflict_txs_ temporary stores conflict information, and will be read by upper layers, bring back to scheduler - ObSArray conflict_txs_; // FARM COMPAT WHITELIST + ObSArray conflict_txs_; // FARM COMPAT WHITELIST for cflict_txs_ ObSArray conflict_info_array_; // used during commit @@ -854,13 +854,20 @@ public: , list_() #endif {} +#ifdef ENABLE_DEBUG_LOG + ~ObTxDescAlloc() + { + ObSpinLockGuard guard(lk_); + list_.remove(); + } +#endif ObTxDesc* alloc_value() { ATOMIC_INC(&alloc_cnt_); ObTxDesc *it = op_alloc(ObTxDesc); #ifdef ENABLE_DEBUG_LOG - ObSpinLockGuard guard(lk_); - list_.insert(it->alloc_link_); + ObSpinLockGuard guard(lk_); + list_.insert(it->alloc_link_); #endif return it; } @@ -875,6 +882,10 @@ public: op_free(v); } } + static void force_free(ObTxDesc *v) + { + op_free(v); + } int64_t get_alloc_cnt() const { return ATOMIC_LOAD(&alloc_cnt_); } #ifdef ENABLE_DEBUG_LOG template @@ -898,6 +909,9 @@ public: ObTxDesc::DLink list_; #endif }; + static void force_release(ObTxDesc &tx) { + ObTxDescAlloc::force_free(&tx); + } share::ObLightHashMap map_; std::function tx_id_allocator_; ObTransService &txs_; diff --git a/src/storage/tx/ob_trans_functor.h b/src/storage/tx/ob_trans_functor.h index 3928a21899..212c3d193a 100644 --- a/src/storage/tx/ob_trans_functor.h +++ b/src/storage/tx/ob_trans_functor.h @@ -357,17 +357,53 @@ private: ObTxCommitCallback *&cb_list_; }; +class FilterTransferTxFunctor +{ +public: + FilterTransferTxFunctor(ObIArray &tablet_list, const SCN data_end_scn, ObIArray &move_tx_ids) : + tablet_list_(tablet_list), data_end_scn_(data_end_scn), + move_tx_ids_(move_tx_ids), count_(0), ret_(OB_SUCCESS) + {} + ~FilterTransferTxFunctor() { PRINT_FUNC_STAT; } + OPERATOR_V4(FilterTransferTxFunctor) + { + bool bool_ret = false; + int ret = OB_SUCCESS; + if (!tx_id.is_valid() || OB_ISNULL(tx_ctx)) { + ret_ = ret = OB_INVALID_ARGUMENT; + TRANS_LOG(WARN, "invalid argument", K(tx_id), "ctx", OB_P(tx_ctx)); + } else { + ++count_; + } + if (OB_SUCC(ret)) { + bool need_transfer = false; + if (OB_FAIL(tx_ctx->check_need_transfer(data_end_scn_, tablet_list_, need_transfer))) { + TRANS_LOG(WARN, "check need transfer failed", KR(ret), K(*tx_ctx)); + ret_ = ret; + } else if (need_transfer && OB_FAIL(move_tx_ids_.push_back(tx_id))) { + ret_ = ret; + } else { + bool_ret = true; + } + } + return bool_ret; + } + int get_ret() const { return ret_; } + int64_t get_count() const { return count_; } +private: + ObIArray &tablet_list_; + const SCN data_end_scn_; + ObIArray &move_tx_ids_; + int64_t count_; + int ret_; +}; + class TransferOutTxOpFunctor { public: - TransferOutTxOpFunctor(const int64_t abs_expired_time, int64_t except_tx_id, const SCN data_end_scn, - const SCN op_scn, NotifyType op_type, bool is_replay, ObLSID dest_ls_id, int64_t transfer_epoch) - : abs_expired_time_(abs_expired_time), except_tx_id_(except_tx_id), data_end_scn_(data_end_scn), - op_scn_(op_scn), op_type_(op_type), is_replay_(is_replay), dest_ls_id_(dest_ls_id), - transfer_epoch_(transfer_epoch), count_(0), op_tx_count_(0), ret_(OB_SUCCESS) + TransferOutTxOpFunctor(const ObTransferOutTxParam ¶m) + : param_(param), count_(0), op_tx_count_(0), ret_(OB_SUCCESS) { - - SET_EXPIRED_LIMIT(100 * 1000 /*100ms*/, 3 * 1000 * 1000 /*3s*/); } ~TransferOutTxOpFunctor() { PRINT_FUNC_STAT; } OPERATOR_V4(TransferOutTxOpFunctor) @@ -379,21 +415,19 @@ public: TRANS_LOG(WARN, "invalid argument", K(tx_id), "ctx", OB_P(tx_ctx)); } else { ++count_; - if ((count_ % BATCH_CHECK_COUNT) == 0) { - const int64_t now = ObTimeUtility::current_time(); - if (now >= abs_expired_time_) { - ret_ = ret = OB_TIMEOUT; - TRANS_LOG(WARN, "transfer block tx timeout", K(count_)); - } - } } if (OB_FAIL(ret)) { - } else if (tx_id.get_id() == except_tx_id_) { + } else if (tx_id.get_id() == param_.except_tx_id_) { bool_ret = true; } else { bool is_operated = false; - if (OB_FAIL(tx_ctx->do_transfer_out_tx_op(data_end_scn_, op_scn_, op_type_, is_replay_, - dest_ls_id_, transfer_epoch_, is_operated))) { + if (OB_FAIL(tx_ctx->do_transfer_out_tx_op(param_.data_end_scn_, + param_.op_scn_, + param_.op_type_, + param_.is_replay_, + param_.dest_ls_id_, + param_.transfer_epoch_, + is_operated))) { TRANS_LOG(WARN, "do_transfer_out_tx_op failed", KR(ret), K(*tx_ctx)); ret_ = ret; } else { @@ -409,15 +443,7 @@ public: int64_t get_count() const { return count_; } int64_t get_op_tx_count() const { return op_tx_count_; } private: - static const int64_t BATCH_CHECK_COUNT = 100; - int64_t abs_expired_time_; - int64_t except_tx_id_; - const SCN data_end_scn_; - const SCN op_scn_; - NotifyType op_type_; - bool is_replay_; - ObLSID dest_ls_id_; - int64_t transfer_epoch_; + const ObTransferOutTxParam ¶m_; int64_t count_; int64_t op_tx_count_; int ret_; @@ -684,13 +710,17 @@ public: tmp_ret = OB_INVALID_ARGUMENT; TRANS_LOG_RET(WARN, tmp_ret, "invalid ls id", K(ls_id), KP(ls_tx_ctx_mgr)); } else { - uint64_t mgr_state = ls_tx_ctx_mgr->get_state_(); + uint64_t mgr_state; + bool is_master = false; + bool is_stopped = true; + const char *state_str = + ls_tx_ctx_mgr->tx_ls_state_mgr_.iter_ctx_mgr_stat_info(mgr_state, is_master, is_stopped); tmp_ret = ls_tx_ctx_mgr_stat.init(addr_, ls_tx_ctx_mgr->ls_id_, - ls_tx_ctx_mgr->is_master_(mgr_state), - ls_tx_ctx_mgr->is_stopped_(mgr_state), + is_master, + is_stopped, mgr_state, - ObLSTxCtxMgr::State::state_str(mgr_state), + state_str, ls_tx_ctx_mgr->total_tx_ctx_count_, (int64_t)(&(*ls_tx_ctx_mgr))); if (OB_SUCCESS != tmp_ret) { diff --git a/src/storage/tx/ob_trans_part_ctx.cpp b/src/storage/tx/ob_trans_part_ctx.cpp index 08016c2601..af160dbff8 100644 --- a/src/storage/tx/ob_trans_part_ctx.cpp +++ b/src/storage/tx/ob_trans_part_ctx.cpp @@ -43,6 +43,7 @@ #undef NEED_MDS_REGISTER_DEFINE #include "storage/tablet/ob_tablet_transfer_tx_ctx.h" #include "storage/tx/ob_ctx_tx_data.h" +#include "share/allocator/ob_shared_memory_allocator_mgr.h" #include "logservice/ob_log_service.h" #include "storage/ddl/ob_ddl_inc_clog_callback.h" #include "storage/tx/ob_tx_log_operator.h" @@ -195,14 +196,18 @@ int ObPartTransCtx::init_for_transfer_move(const ObTxCtxMoveArg &arg) { int ret = OB_SUCCESS; CtxLockGuard guard(lock_); - exec_info_.is_sub2pc_ = arg.is_sub2pc_; - mt_ctx_.set_trans_version(arg.trans_version_); - exec_info_.trans_type_ = TransType::DIST_TRANS; - if (arg.tx_state_ >= ObTxState::PREPARE) { - exec_info_.prepare_version_ = arg.prepare_version_; - ctx_tx_data_.set_commit_version(arg.commit_version_); + if (OB_FAIL(load_tx_op_if_exist_())) { + TRANS_LOG(WARN, "load_tx_op failed", KR(ret), KPC(this)); + } else { + exec_info_.is_sub2pc_ = arg.is_sub2pc_; + mt_ctx_.set_trans_version(arg.trans_version_); + exec_info_.trans_type_ = TransType::DIST_TRANS; + if (arg.tx_state_ >= ObTxState::PREPARE) { + exec_info_.prepare_version_ = arg.prepare_version_; + ctx_tx_data_.set_commit_version(arg.commit_version_); + } + set_durable_state_(arg.tx_state_); } - set_durable_state_(arg.tx_state_); return ret; } @@ -819,12 +824,6 @@ int ObPartTransCtx::commit(const ObTxCommitParts &parts, int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; CtxLockGuard guard(lock_); - if (tenant_id_ % 2 == 0) { - TRANS_LOG(DEBUG, "commit transaction now!", "trans_id", get_trans_id()); - } - if (is_parallel_logging()) { - TRANS_LOG(INFO, "commit transaction now!", KPC(this)); - } if (IS_NOT_INIT) { TRANS_LOG(WARN, "ObPartTransCtx not inited"); ret = OB_NOT_INIT; @@ -1786,6 +1785,10 @@ int ObPartTransCtx::serialize_tx_ctx_to_buffer(ObTxLocalBuffer &buffer, int64_t } else if (OB_FAIL(refresh_rec_log_ts_())) { TRANS_LOG(WARN, "refresh rec log ts failed", K(ret), K(*this)); } else { + SpinRLockManualGuard tx_op_guard; + if (ctx_info.tx_data_guard_.tx_data()->op_guard_.is_valid()) { + tx_op_guard.lock(ctx_info.tx_data_guard_.tx_data()->op_guard_->get_lock()); + } // 6. Do serialize int64_t pos = 0; serialize_size = ctx_info.get_serialize_size(); @@ -2168,6 +2171,9 @@ int ObPartTransCtx::tx_end_(const bool commit) // the user. } else if (OB_FAIL(ctx_tx_data_.set_state(state))) { TRANS_LOG(WARN, "set tx data state failed", K(ret), KPC(this)); + // We need put abort_op into tx_data before trans_end to promise ctx_tx_data is writeable + } else if (!commit && end_scn.is_valid() && OB_FAIL(ctx_tx_data_.add_abort_op(end_scn))) { + TRANS_LOG(WARN, "add tx data abort_op failed", K(ret), KPC(this)); // STEP5: We need invoke mt_ctx_.trans_end after the ctx_tx_data is decided // and filled in because we obey the rule that ObMvccRowCallback::trans_commit // is callbacked from front to back so that if the read or write is standing @@ -2324,6 +2330,174 @@ int ObPartTransCtx::on_success(ObTxLogCb *log_cb) return ret; } +int ObPartTransCtx::replay_mds_to_tx_table_(const ObTxBufferNodeArray &mds_node_array, + const share::SCN op_scn) +{ + int ret = OB_SUCCESS; + ObTxDataGuard tx_data_guard; + ObTxDataGuard new_tx_data_guard; + bool op_exist = false; + if (OB_FAIL(ctx_tx_data_.get_tx_data(tx_data_guard))) { + TRANS_LOG(WARN, "get tx data failed", KR(ret)); + } else if (OB_FAIL(tx_data_guard.tx_data()->check_tx_op_exist(op_scn, op_exist))) { + TRANS_LOG(WARN, "check_tx_op_exist failed", KR(ret)); + } else if (op_exist) { + // do nothing + } else if (OB_FAIL(tx_data_guard.tx_data()->init_tx_op())) { + TRANS_LOG(WARN, "init tx op failed", KR(ret)); + } else { + ObTxOpArray tx_op_batch; + if (OB_FAIL(prepare_mds_tx_op_(mds_node_array, + op_scn, + *tx_data_guard.tx_data()->op_allocator_, + tx_op_batch, + true))) { + TRANS_LOG(WARN, "preapre mds tx_op failed", KR(ret)); + } else if (OB_FAIL(tx_data_guard.tx_data()->op_guard_->add_tx_op_batch(trans_id_, + ls_id_, op_scn, tx_op_batch))) { + TRANS_LOG(WARN, "add_tx_op_batch failed", KR(ret)); + } + } + // tx_ctx and tx_data checkpoint independent + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ls_tx_ctx_mgr_->get_tx_table()->alloc_tx_data(new_tx_data_guard, true, INT64_MAX))){ + TRANS_LOG(WARN, "alloc tx data failed", KR(ret)); + } else { + *new_tx_data_guard.tx_data() = *tx_data_guard.tx_data(); + ObTxData *new_tx_data = new_tx_data_guard.tx_data(); + new_tx_data->end_scn_ = op_scn; + if (OB_FAIL(ls_tx_ctx_mgr_->get_tx_table()->insert(new_tx_data))) { + TRANS_LOG(WARN, "insert tx data failed", KR(ret)); + } + } + TRANS_LOG(INFO, "replay mds to tx_table", KR(ret), K(mds_node_array.count()), K(trans_id_), K(ls_id_), K(op_scn), K(op_exist)); + return ret; +} + +int ObPartTransCtx::insert_mds_to_tx_table_(ObTxLogCb &log_cb) +{ + int ret = OB_SUCCESS; + const ObTxBufferNodeArray &node_array = log_cb.get_mds_range().get_range_array(); + bool all_big_segment = true; + ObTxBufferNodeArray need_process_mds; + for (int64_t idx = 0; OB_SUCC(ret) && idx < node_array.count(); idx++) { + if (!node_array.at(idx).allow_to_use_mds_big_segment()) { + all_big_segment = false; + if (OB_FAIL(need_process_mds.push_back(node_array.at(idx)))) { + TRANS_LOG(WARN, "push to process_mds failed", KR(ret), K(log_cb)); + } + } + } + if (OB_FAIL(ret)) { + } else if (all_big_segment) { + TRANS_LOG(INFO, "MDS big_segment not support tx_op just skip", K(trans_id_), K(ls_id_), KP(this)); + // big segment not support tx_op + if (OB_NOT_NULL(log_cb.get_tx_op_array()) && log_cb.get_tx_op_array()->count() > 0) { + TRANS_LOG(WARN, "MDS big_segment log_cb pre_alloc is not null", KPC(this), K(log_cb)); + } + } else if (OB_ISNULL(log_cb.get_tx_op_array())) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "log_cb tx_op is null", KR(ret), KPC(this), K(log_cb)); + } else if (need_process_mds.count() != log_cb.get_tx_op_array()->count()) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "log_cb mds size is not match", KR(ret), KPC(this), K(log_cb), K(need_process_mds)); + } else { + SCN op_scn = log_cb.get_log_ts(); + ObTxOpArray &tx_op_array = *log_cb.get_tx_op_array(); + ObTxDataGuard tx_data_guard; + // assign mds for pre_alloc node + for (int64_t idx = 0; OB_SUCC(ret) && idx < tx_op_array.count(); idx++) { + tx_op_array.at(idx).set_op_scn(op_scn); + ObTxBufferNodeWrapper &wrapper = *(ObTxBufferNodeWrapper*)(tx_op_array.at(idx).get_op_val()); + if (wrapper.get_node().get_register_no() != need_process_mds.at(idx).get_register_no() || + wrapper.get_node().get_data_source_type() != need_process_mds.at(idx).get_data_source_type()) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "mds not match", KR(ret), KPC(this)); + } else if (OB_FAIL(wrapper.assign(trans_id_, need_process_mds.at(idx), MTL(ObSharedMemAllocMgr*)->tx_data_op_allocator(), true))) { + TRANS_LOG(WARN, "assign mds failed", KR(ret), KPC(this)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ctx_tx_data_.get_tx_data(tx_data_guard))) { + TRANS_LOG(WARN, "get tx data failed", KR(ret)); + } else if (OB_FAIL(tx_data_guard.tx_data()->init_tx_op())) { + TRANS_LOG(WARN, "init tx op failed", KR(ret)); + } else if (OB_FAIL(tx_data_guard.tx_data()->op_guard_->add_tx_op_batch(trans_id_, + ls_id_, op_scn, tx_op_array))) { + TRANS_LOG(WARN, "add_tx_op_batch failed", KR(ret)); + } else { + *log_cb.get_tx_data_guard().tx_data() = *tx_data_guard.tx_data(); + ObTxData *new_tx_data = log_cb.get_tx_data_guard().tx_data(); + new_tx_data->end_scn_ = op_scn; + if (OB_FAIL(ls_tx_ctx_mgr_->get_tx_table()->insert(new_tx_data))) { + TRANS_LOG(WARN, "insert tx data failed", KR(ret)); + } else { + tx_op_array.reset(); + } + } + } + TRANS_LOG(INFO, "insert mds to tx_table", KR(ret), K(trans_id_), K(ls_id_), K(exec_info_.multi_data_source_.count()), K(log_cb)); + return ret; +} + +int ObPartTransCtx::insert_undo_action_to_tx_table_(ObUndoAction &undo_action, + ObTxDataGuard &new_tx_data_guard, + const share::SCN op_scn) +{ + int ret = OB_SUCCESS; + // tx_data on part_ctx has modified + ObTxDataGuard tx_data_guard; + if (OB_FAIL(ctx_tx_data_.get_tx_data(tx_data_guard))) { + TRANS_LOG(WARN, "get tx data failed", KR(ret)); + } else if (OB_FAIL(tx_data_guard.tx_data()->init_tx_op())) { + TRANS_LOG(WARN, "init tx op failed", KR(ret)); + } else if (OB_FAIL(tx_data_guard.tx_data()->add_undo_action(ls_tx_ctx_mgr_->get_tx_table(), undo_action))) { + TRANS_LOG(WARN, "add undo action failed", KR(ret)); + } else { + *new_tx_data_guard.tx_data() = *tx_data_guard.tx_data(); + ObTxData *new_tx_data = new_tx_data_guard.tx_data(); + new_tx_data->end_scn_ = op_scn; + if (OB_FAIL(ls_tx_ctx_mgr_->get_tx_table()->insert(new_tx_data))) { + TRANS_LOG(WARN, "insert tx data failed", KR(ret)); + } + } + TRANS_LOG(INFO, "insert undo_action to tx_table", KR(ret), K(undo_action), K(trans_id_), K(ls_id_), K(op_scn)); + return ret; +} + +int ObPartTransCtx::replay_undo_action_to_tx_table_(ObUndoAction &undo_action, + const share::SCN op_scn) +{ + int ret = OB_SUCCESS; + ObTxDataGuard tx_data_guard; + ObTxDataGuard new_tx_data_guard; + ObTxDataOp *tx_data_op = nullptr; + int64_t tx_data_op_ref = 0; + if (OB_FAIL(ctx_tx_data_.get_tx_data(tx_data_guard))) { + TRANS_LOG(WARN, "get tx data failed", KR(ret)); + } else if (OB_FAIL(tx_data_guard.tx_data()->init_tx_op())) { + TRANS_LOG(WARN, "init tx op failed", KR(ret)); + } else if (OB_FAIL(ls_tx_ctx_mgr_->get_tx_table()->alloc_tx_data(new_tx_data_guard, true, INT64_MAX))){ + TRANS_LOG(WARN, "alloc tx data failed", KR(ret)); + } else { + *new_tx_data_guard.tx_data() = *tx_data_guard.tx_data(); + ObTxData *new_tx_data = new_tx_data_guard.tx_data(); + new_tx_data->end_scn_ = op_scn; + tx_data_op = new_tx_data->op_guard_.ptr(); + if (OB_NOT_NULL(tx_data_op)) { + tx_data_op_ref = tx_data_op->get_ref(); + } + if (OB_FAIL(new_tx_data->add_undo_action(ls_tx_ctx_mgr_->get_tx_table(), undo_action))) { + TRANS_LOG(WARN, "add undo action failed", KR(ret)); + } else if (OB_FAIL(ls_tx_ctx_mgr_->get_tx_table()->insert(new_tx_data))) { + TRANS_LOG(WARN, "insert tx data failed", KR(ret)); + } + } + TRANS_LOG(INFO, "replay undo_action to tx_table", KR(ret), K(undo_action), K(trans_id_), + K(ls_id_), K(op_scn), KP(tx_data_op), K(tx_data_op_ref)); + return ret; +} + int ObPartTransCtx::on_success_ops_(ObTxLogCb *log_cb) { int ret = OB_SUCCESS; @@ -2353,8 +2527,11 @@ int ObPartTransCtx::on_success_ops_(ObTxLogCb *log_cb) false, log_cb->get_mds_range().get_range_array()))) { TRANS_LOG(WARN, "notify data source for ON_REDO", K(ret)); + } else if (OB_FAIL(insert_mds_to_tx_table_(*log_cb))) { + TRANS_LOG(WARN, "inert into tx table failed", KR(ret)); } else { log_cb->get_mds_range().reset(); + log_cb->reset_tx_op_array(); } } else if (ObTxLogType::TX_DIRECT_LOAD_INC_LOG == log_type) { ObTxCtxLogOperator dli_log_op(this, log_cb); @@ -2408,20 +2585,10 @@ int ObPartTransCtx::on_success_ops_(ObTxLogCb *log_cb) TRANS_LOG(INFO, "apply commit info log", KR(ret), K(*this), K(two_phase_log_type)); } } else if (ObTxLogType::TX_ROLLBACK_TO_LOG == log_type) { - ObTxData *tx_data = log_cb->get_tx_data(); - if (OB_ISNULL(tx_data)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "unexpected null ptr", KR(ret), K(*this)); + if (OB_FAIL(insert_undo_action_to_tx_table_(log_cb->get_undo_action(), log_cb->get_tx_data_guard(), log_ts))) { + TRANS_LOG(WARN, "insert to tx table failed", KR(ret), K(*this)); } else { - // although logs may be callbacked out of order, - // insert into tx table all the way, tx table will - // filter out the obsolete one. - tx_data->end_scn_ = log_ts; - if (OB_FAIL(ctx_tx_data_.insert_tmp_tx_data(tx_data))) { - TRANS_LOG(WARN, "insert to tx table failed", KR(ret), K(*this)); - } else { - log_cb->set_tx_data(nullptr); - } + log_cb->set_tx_data(nullptr); } } else if (ObTxLogTypeChecker::is_state_log(log_type)) { sub_state_.clear_state_log_submitting(); @@ -3688,6 +3855,8 @@ int ObPartTransCtx::submit_abort_log_() } } else if (OB_FAIL(acquire_ctx_ref_())) { TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); + } else if (OB_FAIL(ctx_tx_data_.reserve_tx_op_space(1))) { + TRANS_LOG(WARN, "reserve tx_op space failed", KR(ret), KPC(this)); } else if (OB_FAIL(submit_log_block_out_(log_block, SCN::min_scn(), log_cb, replay_hint, barrier, 50 * 1000))) { TRANS_LOG(WARN, "submit log to clog adapter failed", KR(ret), K(*this)); return_log_cb_(log_cb); @@ -4793,7 +4962,7 @@ int ObPartTransCtx::push_replaying_log_ts(const SCN log_ts_ns, const int64_t log } if (OB_UNLIKELY(replay_completeness_.is_unknown())) { const bool replay_continous = exec_info_.next_log_entry_no_ == log_entry_no; - set_replay_completeness_(replay_continous); + set_replay_completeness_(replay_continous, log_ts_ns); } } return ret; @@ -5299,7 +5468,7 @@ int ObPartTransCtx::replay_rollback_to(const ObTxRollbackToLog &log, // all previous log replayed // the txn must not replay from its first log, aka. incomplete-replay TRANS_LOG(INFO, "detect txn replayed from middle", K(ret), K(timestamp), K_(trans_id), K_(ls_id), K_(exec_info)); - set_replay_completeness_(false); + set_replay_completeness_(false, timestamp); ret = OB_SUCCESS; } else if (min_unreplayed_scn > timestamp) { ret = OB_ERR_UNEXPECTED; @@ -6042,6 +6211,8 @@ int ObPartTransCtx::replay_multi_data_source(const ObTxMultiDataSourceLog &log, true, increamental_array))) { TRANS_LOG(WARN, "notify data source for ON_REDO failed", K(ret)); + } else if (OB_FAIL(replay_mds_to_tx_table_(increamental_array, timestamp))) { + TRANS_LOG(WARN, "insert mds_op to tx_table failed", K(ret)); } if (OB_SUCC(ret) && OB_FAIL(check_and_merge_redo_lsns_(lsn))) { @@ -6194,7 +6365,18 @@ int ObPartTransCtx::switch_to_leader(const SCN &start_working_ts) K(contain_mds_table_lock), K(contain_mds_transfer_out), K(need_kill_tx), K(kill_by_append_mode_initial_scn), K(append_mode_initial_scn), KPC(this)); if (OB_FAIL(do_local_tx_end_(TxEndAction::ABORT_TX))) { + //Temporary fix: + //The transaction cannot be killed temporarily, waiting for handle_timeout to retry abort. + //Do not block the rest of the transactions from taking over. + if (OB_TRANS_CANNOT_BE_KILLED == ret) { + TRANS_LOG( + INFO, + "The transaction cannot be killed temporarily, waiting for handle_timeout to retry abort.", + K(ret), KPC(this)); + ret = OB_SUCCESS; + } TRANS_LOG(WARN, "abort tx failed", KR(ret), KPC(this)); + } } else { if (OB_FAIL(do_local_tx_end_(TxEndAction::DELAY_ABORT_TX))) { @@ -6939,7 +7121,6 @@ int ObPartTransCtx::deep_copy_mds_array_(const ObTxBufferNodeArray &mds_array, K(i), K(ctx_array_start_index), K(tmp_buf_arr[i].get_register_no()), K(exec_info_.multi_data_source_[ctx_array_start_index]), KPC(this)); } - } else { if (OB_FAIL(exec_info_.multi_data_source_.push_back(tmp_buf_arr[i]))) { TRANS_LOG(WARN, "push back exec_info_.multi_data_source_ failed", K(ret)); @@ -6953,6 +7134,44 @@ int ObPartTransCtx::deep_copy_mds_array_(const ObTxBufferNodeArray &mds_array, return ret; } +int ObPartTransCtx::prepare_mds_tx_op_(const ObTxBufferNodeArray &mds_array, + SCN op_scn, + ObTenantTxDataOpAllocator &tx_op_allocator, + ObTxOpArray &tx_op_array, + bool is_replay) +{ + int ret = OB_SUCCESS; + int64_t dest_max_register_no = 0; + + for (int64_t i = 0; OB_SUCC(ret) && i < mds_array.count(); i++) { + const ObTxBufferNode &node = mds_array.at(i); + ObTxBufferNodeWrapper *new_node_wrapper = nullptr; + mds::BufferCtx *new_ctx = nullptr; + ObTxOp tx_op; + tx_op_allocator.reset_local_alloc_size(); + if (node.allow_to_use_mds_big_segment()) { + // do nothing + } else if (OB_ISNULL(new_node_wrapper = (ObTxBufferNodeWrapper*)(tx_op_allocator.alloc(sizeof(ObTxBufferNodeWrapper))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + TRANS_LOG(WARN, "allocate memory failed", KR(ret)); + } else if (FALSE_IT(new(new_node_wrapper) ObTxBufferNodeWrapper())) { + } else if (!is_replay && OB_FAIL(new_node_wrapper->pre_alloc(trans_id_, node, tx_op_allocator))) { + TRANS_LOG(WARN, "pre_alloc failed", KR(ret), KPC(this)); + } else if (is_replay && OB_FAIL(new_node_wrapper->assign(trans_id_, node, tx_op_allocator, false))) { + TRANS_LOG(WARN, "assign failed", KR(ret), KPC(this)); + } else if (OB_FAIL(tx_op.init(ObTxOpCode::MDS_OP, op_scn, new_node_wrapper, tx_op_allocator.get_local_alloc_size()))) { + TRANS_LOG(WARN, "init tx_op fail", KR(ret)); + } else if (OB_FAIL(tx_op_array.push_back(tx_op))) { + TRANS_LOG(WARN, "push buffer_node to list fail", KR(ret)); + } + if (OB_FAIL(ret) && OB_NOT_NULL(new_node_wrapper)) { + tx_op_allocator.free(new_node_wrapper); + } + } + TRANS_LOG(INFO, "prepare_mds_tx_op", K(ret), K(trans_id_), K(ls_id_), K(mds_array), K(tx_op_array), K(op_scn)); + return ret; +} + int ObPartTransCtx::decide_state_log_barrier_type_( const ObTxLogType &state_log_type, logservice::ObReplayBarrierType &final_barrier_type) @@ -7057,6 +7276,7 @@ int ObPartTransCtx::submit_multi_data_source_(ObTxLogBlock &log_block) share::SCN mds_base_scn; const int64_t replay_hint = trans_id_.get_id(); ObTxLogCb *log_cb = nullptr; + void *tmp_buf = nullptr; if (mds_cache_.count() > 0) { ObTxMultiDataSourceLog log; ObTxMDSRange range; @@ -7107,7 +7327,20 @@ int ObPartTransCtx::submit_multi_data_source_(ObTxLogBlock &log_block) log_cb = nullptr; } else if ((mds_base_scn.is_valid() ? OB_FALSE_IT(mds_base_scn = share::SCN::scn_inc(mds_base_scn)) : OB_FALSE_IT(mds_base_scn.set_min()))) { - // do nothing + } else if (OB_FAIL(ctx_tx_data_.reserve_tx_op_space(log_cb->get_mds_range().count()))) { + TRANS_LOG(WARN, "reserve tx_op space failed", KR(ret), KPC(this)); + } else if (OB_FAIL(ls_tx_ctx_mgr_->get_tx_table()->alloc_tx_data(log_cb->get_tx_data_guard(), true, INT64_MAX))) { + TRANS_LOG(WARN, "alloc tx_data failed", KR(ret), KPC(this)); + } else if (OB_ISNULL(tmp_buf = mtl_malloc(sizeof(ObTxOpArray), "ObTxOpArray"))) { + TRANS_LOG(WARN, "alloc memory failed", KR(ret), KPC(this)); + } else if (FALSE_IT(new (tmp_buf) ObTxOpArray())) { + } else if (FALSE_IT(log_cb->get_tx_op_array() = (ObTxOpArray*)tmp_buf)) { + } else if (OB_FAIL(prepare_mds_tx_op_(log_cb->get_mds_range().get_range_array(), + SCN::min_scn(), + *log_cb->get_tx_data_guard().tx_data()->op_allocator_, + *log_cb->get_tx_op_array(), + false))) { + TRANS_LOG(WARN, "preapre tx_op failed", KR(ret), KPC(this)); } else if (OB_FAIL(submit_log_block_out_(log_block, mds_base_scn, log_cb, replay_hint, barrier_type))) { TRANS_LOG(WARN, "submit log to clog adapter failed", KR(ret), K(*this)); release_ctx_ref_(); @@ -8011,7 +8244,7 @@ int ObPartTransCtx::sub_end_tx(const int64_t &request_id, return ret; } -int ObPartTransCtx::supplement_undo_actions_if_exist_() +int ObPartTransCtx::supplement_tx_op_if_exist_(const bool for_replay, const SCN replay_scn) { int ret = OB_SUCCESS; @@ -8020,26 +8253,93 @@ int ObPartTransCtx::supplement_undo_actions_if_exist_() ObTxDataGuard tmp_tx_data_guard; tmp_tx_data_guard.reset(); ctx_tx_data_.get_tx_table(tx_table); - const ObTxData *tx_data = nullptr; - if (OB_FAIL(ctx_tx_data_.get_tx_data(guard))) { + if (for_replay && !replay_scn.is_valid()) { + ret = OB_INVALID_ARGUMENT; + TRANS_LOG(WARN, "supplement tx_op", KR(ret), K(for_replay), K(replay_scn), KPC(this)); + } else if (OB_FAIL(ctx_tx_data_.get_tx_data(guard))) { TRANS_LOG(ERROR, "get tx data from ctx tx data failed", KR(ret)); - } else if (OB_NOT_NULL(guard.tx_data()->undo_status_list_.head_)) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "invalid ctx tx data", KR(ret), KPC(guard.tx_data())); - } else if (OB_FAIL(ctx_tx_data_.deep_copy_tx_data_out(tmp_tx_data_guard))) { - TRANS_LOG(WARN, "deep copy tx data in ctx tx data failed.", KR(ret), - K(ctx_tx_data_), KPC(this)); - } else if (OB_FAIL(tx_table->supplement_undo_actions_if_exist( - tmp_tx_data_guard.tx_data()))) { - TRANS_LOG( - WARN, - "supplement undo actions to a tx data when replaying a transaction from the middle failed.", - KR(ret), K(ctx_tx_data_), KPC(this)); + } else if (OB_FAIL(tx_table->alloc_tx_data(tmp_tx_data_guard))) { + TRANS_LOG(WARN, "alloc tx_data failed", KR(ret), KPC(this)); + } else if (FALSE_IT(tmp_tx_data_guard.tx_data()->tx_id_ = trans_id_)) { + } else if (OB_FAIL(tx_table->supplement_tx_op_if_exist(tmp_tx_data_guard.tx_data()))) { + TRANS_LOG(WARN, "supplement tx_op ", KR(ret), K(ctx_tx_data_), KPC(this)); } else if (OB_FAIL(ctx_tx_data_.recover_tx_data(tmp_tx_data_guard.tx_data()))) { TRANS_LOG(WARN, "replace tx data in ctx tx data failed.", KR(ret), K(ctx_tx_data_), KPC(this)); + } else if (for_replay && tmp_tx_data_guard.tx_data()->op_guard_.is_valid() && + OB_FAIL(recover_tx_ctx_from_tx_op_(tmp_tx_data_guard.tx_data()->op_guard_->get_tx_op_list(), replay_scn))) { + TRANS_LOG(WARN, "recover tx_ctx from tx_op failed", KR(ret)); } + TRANS_LOG(INFO, "supplement_tx_op_if_exist_", KR(ret), K(trans_id_), K(ls_id_), K(ctx_tx_data_)); + return ret; +} +int ObPartTransCtx::recover_tx_ctx_from_tx_op_(ObTxOpVector &tx_op_list, const SCN replay_scn) +{ + TRANS_LOG(INFO, "recover_tx_ctx_from_tx_op_", K(tx_op_list.get_count()), K(replay_scn), KPC(this)); + int ret = OB_SUCCESS; + // filter tx_op for this tx_ctx life_cycle + ObTxOpArray ctx_tx_op; + for (int64_t idx = 0; OB_SUCC(ret) && idx < tx_op_list.get_count(); idx++) { + ObTxOp &tx_op = *tx_op_list.at(idx); + if (tx_op.get_op_scn() < replay_scn) { + if (tx_op.get_op_code() == ObTxOpCode::ABORT_OP) { + ctx_tx_op.reuse(); + } else if (OB_FAIL(ctx_tx_op.push_back(tx_op))) { + TRANS_LOG(WARN, "push tx_op to array fail", KR(ret), KPC(this)); + } + } else { + if (tx_op.get_op_code() == ObTxOpCode::ABORT_OP) { + break; + } else if (OB_FAIL(ctx_tx_op.push_back(tx_op))) { + TRANS_LOG(WARN, "push tx_op to array fail", KR(ret), KPC(this)); + } + } + } + // recover tx_op to tx_ctx + ObTxBufferNodeArray mds_array; + for (int64_t idx = 0; OB_SUCC(ret) && idx < ctx_tx_op.count(); idx++) { + ObTxOp &tx_op = ctx_tx_op.at(idx); + if (tx_op.get_op_code() == ObTxOpCode::MDS_OP) { + ObTxBufferNodeWrapper &node_wrapper = *tx_op.get(); + const ObTxBufferNode &node = node_wrapper.get_node(); + if (OB_FAIL(mds_array.push_back(node))) { + TRANS_LOG(WARN, "failed to push node to array", KR(ret), KPC(this)); + } else if (node.type_ == ObTxDataSourceType::TABLE_LOCK) { + // to recover table_lock + ObTableLockOp lock_op; + int64_t pos = 0; + const char *buf = node.data_.ptr(); + const int64_t len = node.data_.length(); + if (OB_FAIL(lock_op.deserialize(buf, node.get_data_size(), pos))) { + TRANS_LOG(WARN, "deserialize fail", KR(ret), KPC(this)); + } else if (OB_FAIL(mt_ctx_.replay_lock(lock_op, tx_op.get_op_scn()))) { + TRANS_LOG(WARN, "recover lock_op failed", KR(ret), KPC(this)); + } else { + TRANS_LOG(INFO, "recover lock_op from tx_op", KPC(this), K(lock_op), K(node_wrapper)); + } + } + } else { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "recover tx_op undefined", KR(ret), KPC(this)); + } + } + ObTxBufferNodeArray _unused_; + if (FAILEDx(deep_copy_mds_array_(mds_array, _unused_))) { + TRANS_LOG(WARN, "deep copy mds array failed", KR(ret), KPC(this)); + } + int64_t mds_max_register_no = 0; + if (mds_array.count() > 0) { + mds_max_register_no = mds_array.at(mds_array.count() - 1).get_register_no(); + } + int64_t ctx_max_register_no = 0; + if (exec_info_.multi_data_source_.count() > 0) { + ctx_max_register_no = exec_info_.multi_data_source_.at(exec_info_.multi_data_source_.count() - 1).get_register_no(); + } + TRANS_LOG(INFO, "recover tx_ctx from tx_op", KR(ret), K(tx_op_list.get_count()), K(ctx_tx_op.count()), + K(mds_array.count()), K(exec_info_.multi_data_source_.count()), + K(mds_max_register_no), K(ctx_max_register_no), + KPC(this)); return ret; } @@ -8348,83 +8648,25 @@ int ObPartTransCtx::rollback_to_savepoint_(const ObTxSEQ from_scn, // _NOTICE_ must load Undo(s) from TxDataTable before overwriten if (replay_completeness_.is_unknown() && !ctx_tx_data_.has_recovered_from_tx_table() && - OB_FAIL(supplement_undo_actions_if_exist_())) { + OB_FAIL(supplement_tx_op_if_exist_(true, replay_scn))) { TRANS_LOG(WARN, "load undos from tx table fail", K(ret), KPC(this)); - } else if (OB_FAIL(ctx_tx_data_.add_undo_action(undo_action))) { - TRANS_LOG(WARN, "recrod undo info fail", K(ret), K(from_scn), K(to_scn), KPC(this)); - } else if (OB_FAIL(ctx_tx_data_.deep_copy_tx_data_out(tmp_tx_data_guard))) { - TRANS_LOG(WARN, "deep copy tx data failed", KR(ret), K(*this)); - } - - // - // when multiple branch-level savepoints were replayed out of order, to ensure - // tx-data with larger end_scn include all undo-actions of others before - // - // we do deleting in frozen memtable and updating (which with largest end_scn) in active memtable - // because distinguish frozen/active memtable is not easy, just always do those two actions. - // - // following is an illusion of this strategy: - // - // assume rollback to logs with scn of: 80 90 110 - // - // and frozen scn is 100 - // - // case 1: replay order: 110, 90, 80 - // case 2: replay order: 110, 80, 90 - // case 3: replay order: 90, 110, 80 - // case 4: replay order: 90, 80, 110 - // - // the operations of each case: - // case 1: insert 110 -> [insert 90, update 110] -> [insert 80, delete 90, udpate 110] - // case 2: insert 110 -> [insert 80 update 110] -> [insert 90, delete 80, update 110] - // case 3: insert 90 -> insert 110 -> [insert 80, delete 90, update 110] - // case 4: insert 90 -> [insert 80, update 90] -> insert 100 - // - - if (OB_SUCC(ret)) { - need_update_tx_data = ctx_tx_data_.get_max_replayed_rollback_scn() > replay_scn; - if (need_update_tx_data && OB_FAIL(ctx_tx_data_.deep_copy_tx_data_out(update_tx_data_guard))) { - TRANS_LOG(WARN, "deep copy tx data failed", KR(ret), K(*this)); - } - } - // prepare end_scn for tx-data items - if (OB_SUCC(ret)) { - tmp_tx_data_guard.tx_data()->end_scn_ = replay_scn; - if (need_update_tx_data) { - // if the tx-data will be inserted into frozen tx-data-memtable, and it may be not the one with largest end_scn - // we must delete others in order to ensure ourself is the valid one with largest end_scn - tmp_tx_data_guard.tx_data()->exclusive_flag_ = ObTxData::ExclusiveType::EXCLUSIVE; - // for update tx-data, use the same end_scn_ - update_tx_data_guard.tx_data()->end_scn_ = ctx_tx_data_.get_max_replayed_rollback_scn(); - update_tx_data_guard.tx_data()->exclusive_flag_ = ObTxData::ExclusiveType::EXCLUSIVE; - } - } - // prepare done, do the final step to insert tx-data-table, this should not fail - if (OB_SUCC(ret)) { - if (OB_FAIL(ctx_tx_data_.insert_tmp_tx_data(tmp_tx_data_guard.tx_data()))) { - TRANS_LOG(WARN, "insert to tx table failed", KR(ret), K(*this)); - } else if (need_update_tx_data && OB_FAIL(ctx_tx_data_.insert_tmp_tx_data(update_tx_data_guard.tx_data()))) { - TRANS_LOG(WARN, "insert to tx table failed", KR(ret), K(*this)); - } - } - // if this is the largest scn replayed, remember it - if (OB_SUCC(ret) && !need_update_tx_data) { - ctx_tx_data_.set_max_replayed_rollback_scn(replay_scn); + } else if (OB_FAIL(replay_undo_action_to_tx_table_(undo_action, replay_scn))) { + TRANS_LOG(WARN, "insert to tx table failed", KR(ret), K(*this)); } } else if (OB_UNLIKELY(exec_info_.max_submitted_seq_no_ > to_scn)) { /* Leader */ - ObUndoAction undo_action(from_scn, to_scn); - ObUndoStatusNode *undo_status = NULL; - if (OB_FAIL(ctx_tx_data_.prepare_add_undo_action(undo_action, tmp_tx_data_guard, undo_status))) { - TRANS_LOG(WARN, "prepare add undo action fail", K(ret), KPC(this)); - } else if (OB_FAIL(submit_rollback_to_log_(from_scn, to_scn, tmp_tx_data_guard.tx_data()))) { + ObTxDataGuard tx_data_guard; + ObTxTable *tx_table = nullptr; + ctx_tx_data_.get_tx_table(tx_table); + ObUndoAction undo(from_scn, to_scn); + if (OB_FAIL(ctx_tx_data_.get_tx_data(tx_data_guard))) { + TRANS_LOG(WARN, "get tx data failed", KR(ret)); + } else if (OB_FAIL(tx_data_guard.tx_data()->init_tx_op())) { + TRANS_LOG(WARN, "init tx op failed", KR(ret)); + } else if (OB_FAIL(tx_data_guard.tx_data()->add_undo_action(ls_tx_ctx_mgr_->get_tx_table(), + undo))) { + TRANS_LOG(WARN, "add undo action failed", KR(ret)); + } else if (OB_FAIL(submit_rollback_to_log_(from_scn, to_scn))) { TRANS_LOG(WARN, "submit undo redolog fail", K(ret), K(from_scn), K(to_scn), KPC(this)); - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(ctx_tx_data_.cancel_add_undo_action(undo_status))) { - TRANS_LOG(ERROR, "cancel add undo action failed", KR(tmp_ret), KPC(this)); - } - } else if (OB_FAIL(ctx_tx_data_.commit_add_undo_action(undo_action, undo_status))) { - TRANS_LOG(ERROR, "oops, commit add undo action fail", K(ret), KPC(this)); - ob_abort(); } } @@ -8441,8 +8683,7 @@ int ObPartTransCtx::rollback_to_savepoint_(const ObTxSEQ from_scn, } int ObPartTransCtx::submit_rollback_to_log_(const ObTxSEQ from_scn, - const ObTxSEQ to_scn, - ObTxData *tx_data) + const ObTxSEQ to_scn) { int ret = OB_SUCCESS; ObTxLogBlock log_block; @@ -8473,6 +8714,8 @@ int ObPartTransCtx::submit_rollback_to_log_(const ObTxSEQ from_scn, TRANS_LOG(ERROR, "cb arg array is empty", K(ret), K(log_block)); return_log_cb_(log_cb); log_cb = NULL; + } else if (OB_FAIL(ls_tx_ctx_mgr_->get_tx_table()->alloc_tx_data(log_cb->get_tx_data_guard(), true, INT64_MAX))) { + TRANS_LOG(WARN, "alloc_tx_data failed", KR(ret), KPC(this)); } else if (OB_FAIL(submit_log_block_out_(log_block, SCN::min_scn(), log_cb, replay_hint, barrier))) { TRANS_LOG(WARN, "submit log fail", K(ret), K(log_block), KPC(this)); return_log_cb_(log_cb); @@ -8481,7 +8724,7 @@ int ObPartTransCtx::submit_rollback_to_log_(const ObTxSEQ from_scn, TRANS_LOG(ERROR, "inc TxCtx ref fail", K(ret), KPC(this)); } else if (OB_FAIL(after_submit_log_(log_block, log_cb, NULL))) { } else { - log_cb->set_tx_data(tx_data); + log_cb->set_undo_action(ObUndoAction(from_scn, to_scn)); } REC_TRANS_TRACE_EXT(tlog_, submit_rollback_log, OB_ID(ret), ret, @@ -8578,7 +8821,9 @@ int ObPartTransCtx::try_alloc_retain_ctx_func_() { int ret = OB_SUCCESS; - if (OB_ISNULL(retain_ctx_func_ptr_)) { + if (is_support_tx_op_()) { + // do nothing + } else if (OB_ISNULL(retain_ctx_func_ptr_)) { if (OB_ISNULL(retain_ctx_func_ptr_ = static_cast( ObTxRetainCtxMgr::alloc_object(sizeof(ObMDSRetainCtxFunctor))))) { @@ -8619,9 +8864,15 @@ int ObPartTransCtx::insert_into_retain_ctx_mgr_(RetainCause cause, if (for_replay) { retain_lock_timeout = 10 * 1000; } + bool need_retain_ctx = !is_support_tx_op_(); + if (need_retain_ctx) { + TRANS_LOG(INFO, "insert into retain_ctx", KPC(this), K(for_replay), K(log_ts)); + } ObTxRetainCtxMgr &retain_ctx_mgr = ls_tx_ctx_mgr_->get_retain_ctx_mgr(); - if (OB_ISNULL(ls_tx_ctx_mgr_) || RetainCause::UNKOWN == cause) { + if (!need_retain_ctx) { + // do nothing + } else if (OB_ISNULL(ls_tx_ctx_mgr_) || RetainCause::UNKOWN == cause) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "invalid argument", K(ret), K(cause), KP(ls_tx_ctx_mgr_), KPC(this)); } else if (OB_ISNULL(retain_ctx_func_ptr_)) { @@ -9598,13 +9849,9 @@ int ObPartTransCtx::collect_tx_ctx(const ObLSID dest_ls_id, TRANS_LOG(INFO, "collect_tx_ctx tx skip ctx exiting", K(trans_id_), K(ls_id_)); } else if (FALSE_IT(start_scn = get_start_log_ts())) { } else if (!start_scn.is_valid() || start_scn > data_end_scn) { - // just for check - if (sub_state_.is_transfer_blocking()) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "tx has transfer_blocking state unexpected", KR(ret), KPC(this), K(start_scn), K(data_end_scn)); - } else { - TRANS_LOG(INFO, "collect_tx_ctx tx skip for start_scn", K(trans_id_), K(ls_id_), K(start_scn.is_valid()), K(start_scn > data_end_scn)); - } + TRANS_LOG(INFO, "collect_tx_ctx tx skip for start_scn", K(trans_id_), K(ls_id_), K(start_scn.is_valid()), K(start_scn > data_end_scn)); + } else if (exec_info_.state_ >= ObTxState::COMMIT) { + TRANS_LOG(INFO, "collect_tx_ctx tx skip ctx has commit", K(trans_id_), K(ls_id_), K(exec_info_.state_)); } else if (!sub_state_.is_transfer_blocking()) { // just for check if (!is_contain_mds_type_(ObTxDataSourceType::START_TRANSFER_OUT_V2)) { @@ -9614,9 +9861,6 @@ int ObPartTransCtx::collect_tx_ctx(const ObLSID dest_ls_id, TRANS_LOG(INFO, "collect_tx_ctx tx skip transfer self", K(trans_id_), K(ls_id_), K(start_scn), K(start_scn > data_end_scn), K(is_contain_mds_type_(ObTxDataSourceType::START_TRANSFER_OUT_V2))); } - } else if (exec_info_.state_ >= ObTxState::COMMIT) { - TRANS_LOG(INFO, "collect_tx_ctx tx skip ctx has commit", K(trans_id_), K(ls_id_), K(exec_info_.state_)); - // filter } else if (sub_state_.is_state_log_submitting()) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "tx is driving when transfer move", KR(ret), KPC(this), @@ -9714,6 +9958,28 @@ int ObPartTransCtx::check_is_aborted_in_tx_data_(const ObTransID tx_id, return ret; } +int ObPartTransCtx::load_tx_op_if_exist_() +{ + int ret = OB_SUCCESS; + ObTxData *tx_data = NULL; + ObTxTableGuard tx_table_guard; + if (OB_FAIL(ctx_tx_data_.get_tx_data_ptr(tx_data))) { + TRANS_LOG(WARN, "get_tx_data failed", KR(ret), KPC(this)); + } else if (OB_ISNULL(tx_data)) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "tx_data is null", KR(ret), KPC(this)); + } else if (OB_FAIL(ls_tx_ctx_mgr_->get_tx_table_guard(tx_table_guard))) { + TRANS_LOG(WARN, "get_tx_table failed", KR(ret), KPC(this)); + } else if (OB_FAIL(tx_table_guard.load_tx_op(trans_id_, *tx_data))) { + if (OB_TRANS_CTX_NOT_EXIST != ret) { + TRANS_LOG(WARN, "load_tx_op failed", KR(ret), KPC(this)); + } else { + ret = OB_SUCCESS; + } + } + return ret; +} + // NB: This function can report a retryable error because the outer while loop // will ignore the error and continuously retry until it succeeds within the // callback function. @@ -9990,20 +10256,21 @@ int ObPartTransCtx::update_tx_data_start_and_end_scn_(const SCN start_scn, TRANS_LOG(WARN, "tx table is null", KR(ret), KPC(this)); } else if (OB_FAIL(ctx_tx_data_.get_tx_data(tx_data_guard))) { TRANS_LOG(WARN, "get tx_data failed", KR(ret)); - } else if (OB_FAIL(tx_table->deep_copy_tx_data(tx_data_guard, tmp_tx_data_guard))) { + } else if (OB_FAIL(tx_table->alloc_tx_data(tmp_tx_data_guard))) { TRANS_LOG(WARN, "copy tx data failed", KR(ret), KPC(this)); } else { - ObTxData *tx_data = tmp_tx_data_guard.tx_data(); + ObTxData *new_tx_data = tmp_tx_data_guard.tx_data(); + *new_tx_data = *tx_data_guard.tx_data(); if (start_scn.is_valid()) { share::SCN current_start_scn = get_start_log_ts(); if (current_start_scn.is_valid()) { - tx_data->start_scn_.atomic_store(MIN(start_scn, current_start_scn)); + new_tx_data->start_scn_.atomic_store(MIN(start_scn, current_start_scn)); } else { - tx_data->start_scn_.atomic_store(start_scn); + new_tx_data->start_scn_.atomic_store(start_scn); } } - tx_data->end_scn_.atomic_store(end_scn); - if (OB_FAIL(tx_table->insert(tx_data))) { + new_tx_data->end_scn_.atomic_store(end_scn); + if (OB_FAIL(tx_table->insert(new_tx_data))) { TRANS_LOG(WARN, "insert tx data failed", KR(ret), KPC(this)); } } @@ -10142,17 +10409,17 @@ inline bool ObPartTransCtx::has_replay_serial_final_() const exec_info_.max_applied_log_ts_ >= exec_info_.serial_final_scn_; } -int ObPartTransCtx::set_replay_incomplete() { +int ObPartTransCtx::set_replay_incomplete(const share::SCN log_ts) { CtxLockGuard guard(lock_); - return set_replay_completeness_(false); + return set_replay_completeness_(false, log_ts); } -int ObPartTransCtx::set_replay_completeness_(const bool complete) +int ObPartTransCtx::set_replay_completeness_(const bool complete, const SCN replay_scn) { int ret = OB_SUCCESS; if (OB_UNLIKELY(replay_completeness_.is_unknown())) { if (!complete && !ctx_tx_data_.has_recovered_from_tx_table()) { - if (OB_FAIL(supplement_undo_actions_if_exist_())) { + if (OB_FAIL(supplement_tx_op_if_exist_(true, replay_scn))) { TRANS_LOG(WARN, "load Undo(s) from tx-table fail", K(ret), KPC(this)); } else { TRANS_LOG(INFO, "replay from middle, load Undo(s) from tx-table succuess", @@ -10175,6 +10442,11 @@ inline bool ObPartTransCtx::is_support_parallel_replay_() const return cluster_version_accurate_ && cluster_version_ >= CLUSTER_VERSION_4_3_0_0; } +inline bool ObPartTransCtx::is_support_tx_op_() const +{ + return cluster_version_accurate_ && cluster_version_ >= CLUSTER_VERSION_4_3_2_0; +} + inline int ObPartTransCtx::switch_to_parallel_logging_(const share::SCN serial_final_scn, const ObTxSEQ max_seq_no) { @@ -10245,5 +10517,44 @@ int ObPartTransCtx::get_stat_for_virtual_table(share::ObLSArray &participants, i return ret; } +int ObPartTransCtx::check_need_transfer( + const SCN data_end_scn, + ObIArray &tablet_list, + bool &need_transfer) +{ + int ret = OB_SUCCESS; + need_transfer = true; + SCN start_scn; + const int64_t LOCK_OP_CHECK_LIMIT = 100; + CtxLockGuard guard(lock_); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "ObPartTransCtx not inited", KR(ret)); + } else if (!data_end_scn.is_valid() || tablet_list.empty()) { + ret = OB_INVALID_ARGUMENT; + TRANS_LOG(WARN, "invalid args", KR(ret), K(data_end_scn), K(tablet_list)); + } else if (FALSE_IT(start_scn = get_start_log_ts())) { + } else if (!start_scn.is_valid() || start_scn > data_end_scn) { + // filter + need_transfer = false; + } else if (exec_info_.state_ >= ObTxState::COMMIT) { + // filter + need_transfer = false; + } else if (mt_ctx_.get_lock_mem_ctx().get_lock_op_count() > LOCK_OP_CHECK_LIMIT) { + // too many lock_op just transfer + } else { + bool contain = false; + for (int64_t idx =0; OB_SUCC(ret) && !contain && idx < tablet_list.count(); idx++) { + if (OB_FAIL(mt_ctx_.get_lock_mem_ctx().check_contain_tablet(tablet_list.at(idx), contain))) { + TRANS_LOG(WARN, "check lock_ctx contain tablet fail", KR(ret), K(tablet_list.at(idx)), K(trans_id_), K(ls_id_)); + } + } + if (OB_SUCC(ret) && !contain) { + need_transfer = false; + } + } + return ret; +} + } // namespace transaction } // namespace oceanbase diff --git a/src/storage/tx/ob_trans_part_ctx.h b/src/storage/tx/ob_trans_part_ctx.h index 307d62e5a7..ef2060957a 100644 --- a/src/storage/tx/ob_trans_part_ctx.h +++ b/src/storage/tx/ob_trans_part_ctx.h @@ -458,7 +458,7 @@ public: void check_no_need_replay_checksum(const share::SCN &log_ts, const int index); bool is_replay_complete_unknown() const { return replay_completeness_.is_unknown(); } - int set_replay_incomplete(); + int set_replay_incomplete(const share::SCN log_ts); // return the min log ts of those logs which are submitted but // not callbacked yet, if there is no such log return INT64_MAX const share::SCN get_min_undecided_log_ts() const; @@ -478,7 +478,8 @@ public: int switch_to_leader(const share::SCN &start_working_ts); int switch_to_follower_gracefully(ObTxCommitCallback *&cb_list); int resume_leader(const share::SCN &start_working_ts); - int supplement_undo_actions_if_exist_(); + int supplement_tx_op_if_exist_(const bool for_replay, const share::SCN replay_scn); + int recover_tx_ctx_from_tx_op_(ObTxOpVector &tx_op_list, const share::SCN replay_scn); void set_role_state(const bool for_replay) { @@ -658,6 +659,15 @@ private: int deep_copy_mds_array_(const ObTxBufferNodeArray &mds_array, ObTxBufferNodeArray &incremental_array, bool need_replace = false); + int prepare_mds_tx_op_(const ObTxBufferNodeArray &mds_array, + share::SCN op_scn, + share::ObTenantTxDataOpAllocator &tx_op_allocator, + ObTxOpArray &tx_op_list, + bool is_replay); + int replay_mds_to_tx_table_(const ObTxBufferNodeArray &mds_node_array, const share::SCN op_scn); + int insert_mds_to_tx_table_(ObTxLogCb &log_cb); + int insert_undo_action_to_tx_table_(ObUndoAction &undo_action, ObTxDataGuard &new_tx_data_guard, const share::SCN op_scn); + int replay_undo_action_to_tx_table_(ObUndoAction &undo_action, const share::SCN op_scn); int decide_state_log_barrier_type_(const ObTxLogType &state_log_type, logservice::ObReplayBarrierType &final_barrier_type); bool is_contain_mds_type_(const ObTxDataSourceType target_type); @@ -684,8 +694,9 @@ private: const share::SCN ×tamp, const int64_t &part_log_no); bool is_support_parallel_replay_() const; - int set_replay_completeness_(const bool complete); + int set_replay_completeness_(const bool complete, const share::SCN replay_scn); int errsim_notify_mds_(); + bool is_support_tx_op_() const; protected: virtual int get_gts_(share::SCN >s); virtual int wait_gts_elapse_commit_version_(bool &need_wait); @@ -805,6 +816,9 @@ private: // ======================= for transfer =============================== public: + int check_need_transfer(const share::SCN data_end_scn, + ObIArray &tablet_list, + bool &need_transfer); int do_transfer_out_tx_op(const share::SCN data_end_scn, const share::SCN op_scn, const NotifyType op_type, @@ -825,6 +839,7 @@ public: bool is_exec_complete_without_lock(ObLSID ls_id, int64_t epoch, int64_t transfer_epoch); private: int transfer_op_log_cb_(share::SCN op_scn, NotifyType op_type); + int load_tx_op_if_exist_(); int update_tx_data_start_and_end_scn_(const share::SCN start_scn, const share::SCN end_scn, const share::SCN transfer_scn); @@ -918,8 +933,7 @@ private: const ObTxSEQ to_scn, const share::SCN replay_scn = share::SCN::invalid_scn()); int submit_rollback_to_log_(const ObTxSEQ from_scn, - const ObTxSEQ to_scn, - ObTxData *tx_data); + const ObTxSEQ to_scn); int set_state_info_array_(); int update_state_info_array_(const ObStateInfo& state_info); int update_state_info_array_with_transfer_parts_(const ObTxCommitParts &parts, const ObLSID &ls_id); diff --git a/src/storage/tx/ob_trans_service_v4.cpp b/src/storage/tx/ob_trans_service_v4.cpp index 9faab0267a..903769ef00 100644 --- a/src/storage/tx/ob_trans_service_v4.cpp +++ b/src/storage/tx/ob_trans_service_v4.cpp @@ -3932,5 +3932,15 @@ int ObTransService::handle_ask_tx_state_for_4377(const ObAskTxStateFor4377Msg &m TRANS_LOG(INFO, "handle ask tx state for 4377", K(ret), K(msg), K(is_alive)); return ret; } + +void ObTransService::force_release_tx_when_session_destroy(ObTxDesc &tx) +{ + { + ObSpinLockGuard guard(tx.lock_); + TRANS_LOG_RET(WARN, OB_SUCCESS, "txdesc will be released forcedly", K(tx)); + tx.print_trace_(); + } + ObTxDescMgr::force_release(tx); +} } // transaction } // ocenabase diff --git a/src/storage/tx/ob_trans_service_v4.h b/src/storage/tx/ob_trans_service_v4.h index a8a0614d94..f49b7910cb 100644 --- a/src/storage/tx/ob_trans_service_v4.h +++ b/src/storage/tx/ob_trans_service_v4.h @@ -21,6 +21,11 @@ int remove_ls(const share::ObLSID &ls_id, */ int acquire_tx(const char* buf, const int64_t len, int64_t &pos, ObTxDesc *&tx); int release_tx_ref(ObTxDesc &tx); +/* + * used when session destroyed + * and TxDesc which referenced by session and the tenant unit has removed + */ +static void force_release_tx_when_session_destroy(ObTxDesc &tx); /* * interrupt any work in progress thread */ @@ -381,7 +386,6 @@ int rollback_to_local_implicit_savepoint_(ObTxDesc &tx, int rollback_to_global_implicit_savepoint_(ObTxDesc &tx, const ObTxSEQ savepoint, const int64_t expire_ts, - const share::ObLSArray *extra_touched_ls, const int exec_errcode); int ls_sync_rollback_savepoint__(ObPartTransCtx *part_ctx, const ObTxSEQ savepoint, @@ -393,8 +397,7 @@ int ls_sync_rollback_savepoint__(ObPartTransCtx *part_ctx, ObIArray &downstream_parts); void tx_post_terminate_(ObTxDesc &tx); int start_epoch_(ObTxDesc &tx); -// in_stmt means stmt is executing -int tx_sanity_check_(ObTxDesc &tx, const bool in_stmt = false); +int tx_sanity_check_(ObTxDesc &tx); bool tx_need_reset_(const int error_code) const; int get_tx_table_guard_(ObLS *ls, const share::ObLSID &ls_id, diff --git a/src/storage/tx/ob_trans_submit_log_cb.cpp b/src/storage/tx/ob_trans_submit_log_cb.cpp index 44c63bee34..a79fd8b947 100644 --- a/src/storage/tx/ob_trans_submit_log_cb.cpp +++ b/src/storage/tx/ob_trans_submit_log_cb.cpp @@ -93,6 +93,18 @@ int ObTxLogCb::init(const ObLSID &key, return ret; } +void ObTxLogCb::reset_tx_op_array() +{ + if (OB_NOT_NULL(tx_op_array_)) { + for (int64_t idx = 0; idx < tx_op_array_->count(); idx++) { + tx_op_array_->at(idx).release(); + } + tx_op_array_->~ObTxOpArray(); + mtl_free(tx_op_array_); + tx_op_array_ = nullptr; + } +} + void ObTxLogCb::reset() { ObTxBaseLogCb::reset(); @@ -115,6 +127,7 @@ void ObTxLogCb::reset() // is_callbacking_ = false; first_part_scn_.invalid_scn(); + reset_tx_op_array(); } void ObTxLogCb::reuse() @@ -132,6 +145,7 @@ void ObTxLogCb::reuse() need_free_extra_cb_ = false; first_part_scn_.invalid_scn(); + reset_tx_op_array(); } ObTxLogType ObTxLogCb::get_last_log_type() const diff --git a/src/storage/tx/ob_trans_submit_log_cb.h b/src/storage/tx/ob_trans_submit_log_cb.h index ac3439918f..9e3f957b4a 100644 --- a/src/storage/tx/ob_trans_submit_log_cb.h +++ b/src/storage/tx/ob_trans_submit_log_cb.h @@ -28,6 +28,7 @@ #include "storage/tx/ob_tx_log.h" #include "storage/memtable/mvcc/ob_mvcc_trans_ctx.h" #include "storage/memtable/ob_redo_log_generator.h" +#include "storage/tx/ob_tx_data_op.h" namespace oceanbase { @@ -74,13 +75,14 @@ class ObTxLogCb : public ObTxBaseLogCb, public common::ObDLinkBase { public: - ObTxLogCb() : extra_cb_(nullptr), need_free_extra_cb_(false) { reset(); } + ObTxLogCb() : extra_cb_(nullptr), need_free_extra_cb_(false), tx_op_array_(nullptr) { reset(); } ~ObTxLogCb() { destroy(); } int init(const share::ObLSID &key, const ObTransID &trans_id, ObTransCtx *ctx, const bool is_dynamic); void reset(); + void reset_tx_op_array(); void reuse(); void destroy() { reset(); } ObTxLogType get_last_log_type() const; @@ -93,7 +95,13 @@ public: tx_data_guard_.init(tx_data); } } + void set_undo_action(const ObUndoAction &undo_action) { + undo_action_ = undo_action; + } + ObUndoAction &get_undo_action() { return undo_action_; } + ObTxOpArray *&get_tx_op_array() { return tx_op_array_; } ObTxData* get_tx_data() { return tx_data_guard_.tx_data(); } + ObTxDataGuard &get_tx_data_guard() { return tx_data_guard_; } int set_callbacks(const ObCallbackScopeArray &callbacks) { return callbacks_.assign(callbacks); } ObCallbackScopeArray& get_callbacks() { return callbacks_; } int reserve_callbacks(int cnt) { return callbacks_.reserve(cnt); } @@ -161,6 +169,8 @@ private: storage::ObDDLIncLogBasic dli_batch_key_; logservice::AppendCb * extra_cb_; bool need_free_extra_cb_; + ObUndoAction undo_action_; + storage::ObTxOpArray *tx_op_array_; //bool is_callbacking_; }; diff --git a/src/storage/tx/ob_tx_api.cpp b/src/storage/tx/ob_tx_api.cpp index 3ccf6569f9..f49f21d778 100644 --- a/src/storage/tx/ob_tx_api.cpp +++ b/src/storage/tx/ob_tx_api.cpp @@ -509,6 +509,11 @@ int ObTransService::submit_commit_tx(ObTxDesc &tx, abort_tx_(tx, ObTxAbortCause::PARTICIPANT_STATE_INCOMPLETE); handle_tx_commit_result_(tx, OB_TRANS_ROLLBACKED); ret = OB_TRANS_ROLLBACKED; + } else if (tx.flags_.PART_ABORTED_) { + TRANS_LOG(WARN, "txn participant aborted, can not commit", K(ret), K(tx)); + abort_tx_(tx, OB_TRANS_ROLLBACKED); + handle_tx_commit_result_(tx, OB_TRANS_ROLLBACKED); + ret = OB_TRANS_ROLLBACKED; } else { int clean = true; ARRAY_FOREACH_X(tx.parts_, i, cnt, clean) { @@ -963,7 +968,7 @@ int ObTransService::create_implicit_savepoint(ObTxDesc &tx, // TODO: rework this interface, allow skip pass tx_param if not required ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "tx param invalid", K(ret), K(tx_param), K(tx)); - } else if (OB_FAIL(tx_sanity_check_(tx, !release))) { + } else if (OB_FAIL(tx_sanity_check_(tx))) { } else if (tx.state_ >= ObTxDesc::State::IN_TERMINATE) { ret = OB_TRANS_INVALID_STATE; TRANS_LOG(WARN, "create implicit savepoint but tx terminated", K(ret), K(tx)); @@ -1067,10 +1072,11 @@ int ObTransService::rollback_to_implicit_savepoint(ObTxDesc &tx, { int ret = OB_SUCCESS; ObSpinLockGuard guard(tx.lock_); - if (OB_FAIL(tx_sanity_check_(tx, true))) { - } else if (savepoint.get_branch() // NOTE: branch savepoint only support local rollback + + if (savepoint.get_branch() // NOTE: branch savepoint only support local rollback || tx.flags_.SHADOW_) { - if (OB_NOT_NULL(extra_touched_ls)) { + if (OB_FAIL(tx_sanity_check_(tx))) { + } else if (OB_NOT_NULL(extra_touched_ls)) { ret = OB_NOT_SUPPORTED; TRANS_LOG(WARN, "rollback on remote only suport collected tx parts", K(ret), K(savepoint), K(tx)); @@ -1078,11 +1084,24 @@ int ObTransService::rollback_to_implicit_savepoint(ObTxDesc &tx, ret = rollback_to_local_implicit_savepoint_(tx, savepoint, expire_ts); } } else { - ret = rollback_to_global_implicit_savepoint_(tx, - savepoint, - expire_ts, - extra_touched_ls, - exec_errcode); + if (tx.state_ < ObTxDesc::State::IN_TERMINATE) { + if (OB_NOT_NULL(extra_touched_ls) && !extra_touched_ls->empty()) { + if (OB_FAIL(tx.update_parts(*extra_touched_ls))) { + TRANS_LOG(WARN, "add tx part with extra_touched_ls fail", K(ret), K(tx), KPC(extra_touched_ls)); + abort_tx_(tx, ret); + } else { + TRANS_LOG(INFO, "add tx part with extra_touched_ls", KPC(extra_touched_ls), K_(tx.tx_id)); + } + } + } + + if (OB_FAIL(tx_sanity_check_(tx))) { + } else { + ret = rollback_to_global_implicit_savepoint_(tx, + savepoint, + expire_ts, + exec_errcode); + } } return ret; } @@ -1164,23 +1183,12 @@ int ObTransService::rollback_to_local_implicit_savepoint_(ObTxDesc &tx, int ObTransService::rollback_to_global_implicit_savepoint_(ObTxDesc &tx, const ObTxSEQ savepoint, const int64_t expire_ts, - const share::ObLSArray *extra_touched_ls, const int exec_errcode) { int ret = OB_SUCCESS; int64_t start_ts = ObTimeUtility::current_time(); tx.inc_op_sn(); bool reset_tx = false, normal_rollback = false, reset_active_scn = false; - // merge extra touched ls - if (OB_NOT_NULL(extra_touched_ls) && !extra_touched_ls->empty()) { - if (OB_FAIL(tx.update_parts(*extra_touched_ls))) { - TRANS_LOG(WARN, "add tx part with extra_touched_ls fail", K(ret), K(tx), KPC(extra_touched_ls)); - abort_tx_(tx, ret); - } else { - TRANS_LOG(INFO, "add tx part with extra_touched_ls", KPC(extra_touched_ls), K_(tx.tx_id)); - } - } - if (OB_SUCC(ret)) { switch(tx.state_) { case ObTxDesc::State::IDLE: @@ -1236,7 +1244,7 @@ int ObTransService::rollback_to_global_implicit_savepoint_(ObTxDesc &tx, if (reset_tx) { } else if (OB_FAIL(ret)) { TRANS_LOG(WARN, "rollback savepoint fail, abort tx", - K(ret), K(savepoint), KP(extra_touched_ls), K(parts), K(tx)); + K(ret), K(savepoint), K(parts), K(tx)); // advance op_sequence to reject further rollback resp messsages tx.inc_op_sn(); abort_tx_(tx, ObTxAbortCause::SAVEPOINT_ROLLBACK_FAIL); @@ -1277,7 +1285,6 @@ int ObTransService::rollback_to_global_implicit_savepoint_(ObTxDesc &tx, REC_TRANS_TRACE_EXT(&tlog, rollback_global_implicit_savepoint, OB_Y(ret), OB_ID(savepoint), savepoint.cast_to_int(), OB_Y(expire_ts), OB_ID(time_used), elapsed_us, - OB_ID(arg), (void*)extra_touched_ls, OB_ID(tag1), reset_tx, OB_ID(opid), tx.op_sn_, OB_ID(ref), tx.get_ref(), @@ -2014,7 +2021,7 @@ int ObTransService::release_tx_ref(ObTxDesc &tx) return tx_desc_mgr_.release_tx_ref(&tx); } -OB_INLINE int ObTransService::tx_sanity_check_(ObTxDesc &tx, const bool in_stmt) +OB_INLINE int ObTransService::tx_sanity_check_(ObTxDesc &tx) { int ret = OB_SUCCESS; if (tx.expire_ts_ <= ObClockGenerator::getClock()) { @@ -2029,14 +2036,8 @@ OB_INLINE int ObTransService::tx_sanity_check_(ObTxDesc &tx, const bool in_stmt) case ObTxDesc::State::ACTIVE: case ObTxDesc::State::IMPLICIT_ACTIVE: if (tx.flags_.PART_ABORTED_) { - if (!in_stmt) { - // not inside of stmt(stmt has not began to execute), abort tx now - TRANS_LOG(WARN, "some participant was aborted, abort tx now"); - abort_tx_(tx, tx.abort_cause_); - } else { - // abort tx after stmt has executed - TRANS_LOG(WARN, "some participant was aborted, but inside stmt, not abort tx now"); - } + TRANS_LOG(WARN, "some participant was aborted, abort tx now"); + abort_tx_(tx, tx.abort_cause_); // go through } else { break; diff --git a/src/storage/tx/ob_tx_data_define.cpp b/src/storage/tx/ob_tx_data_define.cpp index a342edd8a4..fc1b85f435 100644 --- a/src/storage/tx/ob_tx_data_define.cpp +++ b/src/storage/tx/ob_tx_data_define.cpp @@ -14,8 +14,11 @@ #include "lib/utility/ob_unify_serialize.h" #include "storage/tx_table/ob_tx_table.h" #include "share/rc/ob_tenant_base.h" +#include "share/allocator/ob_shared_memory_allocator_mgr.h" +#include "storage/tx/ob_tx_data_op.h" using namespace oceanbase::share; +using namespace oceanbase::transaction; namespace oceanbase { @@ -267,6 +270,7 @@ const char* ObTxCommitData::get_state_string(int32_t state) int ObTxData::serialize(char *buf, const int64_t buf_len, int64_t &pos) const { int ret = OB_SUCCESS; + int64_t pos_tmp = 0; const int64_t len = get_serialize_size_(); if (OB_UNLIKELY(OB_ISNULL(buf) || buf_len <= 0 || pos > buf_len)) { @@ -277,8 +281,9 @@ int ObTxData::serialize(char *buf, const int64_t buf_len, int64_t &pos) const K(pos)); } else if (OB_FAIL(serialization::encode_vi64(buf, buf_len, pos, len))) { STORAGE_LOG(WARN, "encode length of ObTxData failed.", KR(ret), KP(buf), K(buf_len), K(pos)); - } else if (OB_FAIL(serialize_(buf, buf_len, pos))) { - STORAGE_LOG(WARN, "serialize_ of ObTxData failed.", KR(ret), KP(buf), K(buf_len), K(pos)); + } else if (FALSE_IT(pos_tmp = pos)) { + } else if (OB_FAIL(serialize_(buf, pos + len, pos))) { + STORAGE_LOG(WARN, "serialize_ of ObTxData failed.", KR(ret), KP(buf), K(buf_len), K(pos), K(pos_tmp)); } return ret; } @@ -298,10 +303,28 @@ int ObTxData::serialize_(char *buf, const int64_t buf_len, int64_t &pos) const STORAGE_LOG(WARN, "serialize start_scn fail.", KR(ret), K(pos), K(buf_len)); } else if (OB_FAIL(end_scn_.serialize(buf, buf_len, pos))) { STORAGE_LOG(WARN, "serialize end_scn fail.", KR(ret), K(pos), K(buf_len)); - } else if (OB_FAIL(undo_status_list_.serialize(buf, buf_len, pos))) { - STORAGE_LOG(WARN, "serialize undo_status_list fail.", KR(ret), K(pos), K(buf_len)); } - + uint64_t data_version = 0; + if (FAILEDx(GET_MIN_DATA_VERSION(MTL_ID(), data_version))) { + STORAGE_LOG(WARN, "fail to get data version", KR(ret)); + } else if (data_version < DATA_VERSION_4_3_2_0) { + if (op_guard_.is_valid()) { + if (OB_FAIL(op_guard_->get_undo_status_list().serialize(buf, buf_len, pos))) { + STORAGE_LOG(WARN, "serialize undo_status_list fail.", KR(ret), K(pos), K(buf_len)); + } + } else { + ObUndoStatusList dummy_undo; + if (OB_FAIL(dummy_undo.serialize(buf, buf_len, pos))) { + STORAGE_LOG(WARN, "serialize undo_status_list fail.", KR(ret), K(pos), K(buf_len)); + } + } + } else if (op_guard_.is_valid()) { + if (OB_FAIL(op_guard_->get_undo_status_list().serialize(buf, buf_len, pos))) { + STORAGE_LOG(WARN, "serialize undo_status_list fail.", KR(ret), K(pos), K(buf_len)); + } else if (OB_FAIL(op_guard_->get_tx_op_list().serialize(buf, buf_len, pos))) { + STORAGE_LOG(WARN, "serialize tx_op_list fail.", KR(ret), K(pos), K(buf_len)); + } + } return ret; } @@ -324,13 +347,32 @@ int64_t ObTxData::get_serialize_size_() const len += commit_version_.get_serialize_size(); len += start_scn_.get_serialize_size(); len += end_scn_.get_serialize_size(); - len += undo_status_list_.get_serialize_size(); + int ret = OB_SUCCESS; + uint64_t data_version = 0; + if (OB_FAIL(GET_MIN_DATA_VERSION(MTL_ID(), data_version))) { + STORAGE_LOG(ERROR, "get data_version failed", KR(ret)); + } + if (data_version < DATA_VERSION_4_3_2_0) { + if (op_guard_.is_valid()) { + len += op_guard_->get_undo_status_list().get_serialize_size(); + } else { + ObUndoStatusList dummy_undo; + len += dummy_undo.get_serialize_size(); + } + } else if (op_guard_.is_valid()) { + len += op_guard_->get_undo_status_list().get_serialize_size(); + len += op_guard_->get_tx_op_list().get_serialize_size(); + } return len; } -int64_t ObTxData::size() const +int64_t ObTxData::size_need_cache() const { - int64_t len = (TX_DATA_SLICE_SIZE * (1LL + undo_status_list_.undo_node_cnt_)); + int64_t len = TX_DATA_SLICE_SIZE; + if (op_guard_.is_valid()) { + len += TX_DATA_SLICE_SIZE; // tx_op + len += TX_DATA_SLICE_SIZE * op_guard_->get_undo_status_list().undo_node_cnt_; + } return len; } @@ -342,6 +384,7 @@ int ObTxData::deserialize(const char *buf, int ret = OB_SUCCESS; int64_t version = 0; int64_t len = 0; + int64_t pos_tmp = 0; if (OB_UNLIKELY(nullptr == buf || data_len <= 0 || pos > data_len)) { ret = OB_INVALID_ARGUMENT; @@ -356,8 +399,9 @@ int ObTxData::deserialize(const char *buf, } else if (OB_UNLIKELY(pos + len > data_len)) { ret = OB_INVALID_SIZE; STORAGE_LOG(WARN, "length from deserialize is invalid.", KR(ret), K(pos), K(len), K(data_len)); - } else if (OB_FAIL(deserialize_(buf, data_len, pos, slice_allocator))) { - STORAGE_LOG(WARN, "deserialize tx data failed.", KR(ret)); + } else if (FALSE_IT(pos_tmp = pos)) { + } else if (OB_FAIL(deserialize_(buf, pos + len, pos, slice_allocator))) { + STORAGE_LOG(WARN, "deserialize tx data failed.", KR(ret), K(buf), K(pos), K(len), K(pos_tmp), K(data_len)); } return ret; @@ -380,10 +424,17 @@ int ObTxData::deserialize_(const char *buf, STORAGE_LOG(WARN, "deserialize start_scn fail.", KR(ret), K(pos), K(data_len)); } else if (OB_FAIL(end_scn_.deserialize(buf, data_len, pos))) { STORAGE_LOG(WARN, "deserialize end_scn fail.", KR(ret), K(pos), K(data_len)); - } else if (OB_FAIL(undo_status_list_.deserialize(buf, data_len, pos, tx_data_allocator))) { - STORAGE_LOG(WARN, "deserialize undo_status_list fail.", KR(ret), K(pos), K(data_len)); } - + if (OB_SUCC(ret) && pos < data_len) { + if (OB_FAIL(init_tx_op())) { + STORAGE_LOG(WARN, "init tx op fail", KR(ret)); + } else if (OB_FAIL(op_guard_->get_undo_status_list().deserialize(buf, data_len, pos, tx_data_allocator))) { + STORAGE_LOG(WARN, "deserialize undo_status_list fail.", KR(ret), K(pos), K(data_len)); + } else if (pos < data_len && OB_FAIL(op_guard_->get_tx_op_list().deserialize(buf, data_len, pos, + MTL(ObSharedMemAllocMgr*)->tx_data_op_allocator()))) { + STORAGE_LOG(WARN, "deserialize tx_op_list fail.", KR(ret), K(pos), K(data_len)); + } + } return ret; } @@ -396,9 +447,9 @@ void ObTxData::reset() ob_abort(); } ObTxCommitData::reset(); + op_guard_.reset(); tx_data_allocator_ = nullptr; ref_cnt_ = 0; - undo_status_list_.reset(); } ObTxData::ObTxData(const ObTxData &rhs) @@ -413,7 +464,9 @@ ObTxData &ObTxData::operator=(const ObTxData &rhs) commit_version_ = rhs.commit_version_; start_scn_ = rhs.start_scn_; end_scn_ = rhs.end_scn_; - undo_status_list_ = rhs.undo_status_list_; + if (rhs.op_guard_.is_valid()) { + op_guard_.init(rhs.op_guard_.ptr()); + } return *this; } @@ -424,7 +477,7 @@ ObTxData &ObTxData::operator=(const ObTxCommitData &rhs) commit_version_ = rhs.commit_version_; start_scn_ = rhs.start_scn_; end_scn_ = rhs.end_scn_; - undo_status_list_.reset(); + op_guard_.reset(); return *this; } @@ -441,9 +494,6 @@ bool ObTxData::is_valid_in_tx_data_table() const if (!end_scn_.is_valid()) { bool_ret = false; STORAGE_LOG_RET(ERROR, OB_INVALID_ERROR, "tx data end log ts is invalid", KPC(this)); - } else if (OB_ISNULL(undo_status_list_.head_)) { - bool_ret = false; - STORAGE_LOG_RET(ERROR, OB_INVALID_ERROR, "tx data undo status list is invalid", KPC(this)); } else { bool_ret = true; } @@ -468,54 +518,91 @@ bool ObTxData::is_valid_in_tx_data_table() const return bool_ret; } -int ObTxData::add_undo_action(ObTxTable *tx_table, transaction::ObUndoAction &new_undo_action, ObUndoStatusNode *undo_node) +int ObTxData::reserve_undo(ObTxTable *tx_table) { - // STORAGE_LOG(DEBUG, "do add_undo_action"); int ret = OB_SUCCESS; - SpinWLockGuard guard(undo_status_list_.lock_); ObTxDataTable *tx_data_table = nullptr; - ObUndoStatusNode *node = undo_status_list_.head_; if (OB_ISNULL(tx_table)) { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "tx table is nullptr.", KR(ret)); } else if (OB_ISNULL(tx_data_table = tx_table->get_tx_data_table())) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "tx data table in tx table is nullptr.", KR(ret)); - } else if (OB_FAIL(merge_undo_actions_(tx_data_table, node, new_undo_action))) { - STORAGE_LOG(WARN, "merge undo actions fail.", KR(ret), K(new_undo_action)); - } else if (!new_undo_action.is_valid()) { - // if new_undo_action is merged, it will be set to invalid and skip insert + } else if (!op_guard_.is_valid()) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "tx data op is nullptr", KR(ret)); } else { - // generate new node if current node cannot be inserted + SpinWLockGuard lock_guard(op_guard_->get_lock()); + SpinWLockGuard guard(op_guard_->get_undo_status_list().lock_); + ObUndoStatusNode *node = op_guard_->get_undo_status_list().head_; if (OB_ISNULL(node) || node->size_ >= TX_DATA_UNDO_ACT_MAX_NUM_PER_NODE) { ObUndoStatusNode *new_node = nullptr; - if (OB_NOT_NULL(undo_node)) { - new_node = undo_node; - undo_node = NULL; - } else if (OB_FAIL(tx_data_table->alloc_undo_status_node(new_node))) { + if (OB_FAIL(tx_data_table->alloc_undo_status_node(new_node))) { STORAGE_LOG(WARN, "alloc_undo_status_node() fail", KR(ret)); - } - - if (OB_SUCC(ret)) { - new_node->next_ = node; - undo_status_list_.head_ = new_node; - node = new_node; - undo_status_list_.undo_node_cnt_++; - } - } - - if (OB_SUCC(ret)) { - if (OB_NOT_NULL(node)) { - node->undo_actions_[node->size_++] = new_undo_action; } else { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "node is unexpected nullptr", KR(ret), KPC(this)); + new_node->next_ = node; + op_guard_->get_undo_status_list().head_ = new_node; + op_guard_->get_undo_status_list().undo_node_cnt_++; } } } + return ret; +} - if (OB_NOT_NULL(undo_node)) { - tx_data_table->free_undo_status_node(undo_node); +int ObTxData::add_undo_action(ObTxTable *tx_table, transaction::ObUndoAction &new_undo_action, ObUndoStatusNode *undo_node) +{ + // STORAGE_LOG(DEBUG, "do add_undo_action"); + int ret = OB_SUCCESS; + ObTxDataTable *tx_data_table = nullptr; + if (OB_ISNULL(tx_table)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "tx table is nullptr.", KR(ret)); + } else if (OB_ISNULL(tx_data_table = tx_table->get_tx_data_table())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "tx data table in tx table is nullptr.", KR(ret)); + } else if (!op_guard_.is_valid()) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "tx data op is nullptr", KR(ret)); + } else { + SpinWLockGuard lock_guard(op_guard_->get_lock()); + SpinWLockGuard guard(op_guard_->get_undo_status_list().lock_); + ObUndoStatusNode *node = op_guard_->get_undo_status_list().head_; + if (OB_FAIL(merge_undo_actions_(tx_data_table, node, new_undo_action))) { + STORAGE_LOG(WARN, "merge undo actions fail.", KR(ret), K(new_undo_action)); + } else if (!new_undo_action.is_valid()) { + // if new_undo_action is merged, it will be set to invalid and skip insert + } else { + // generate new node if current node cannot be inserted + if (OB_ISNULL(node) || node->size_ >= TX_DATA_UNDO_ACT_MAX_NUM_PER_NODE) { + ObUndoStatusNode *new_node = nullptr; + if (OB_NOT_NULL(undo_node)) { + new_node = undo_node; + undo_node = NULL; + } else if (OB_FAIL(tx_data_table->alloc_undo_status_node(new_node))) { + STORAGE_LOG(WARN, "alloc_undo_status_node() fail", KR(ret)); + } + + if (OB_SUCC(ret)) { + new_node->next_ = node; + op_guard_->get_undo_status_list().head_ = new_node; + node = new_node; + op_guard_->get_undo_status_list().undo_node_cnt_++; + } + } + + if (OB_SUCC(ret)) { + if (OB_NOT_NULL(node)) { + node->undo_actions_[node->size_++] = new_undo_action; + } else { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "node is unexpected nullptr", KR(ret), KPC(this)); + } + } + } + + if (OB_NOT_NULL(undo_node)) { + tx_data_table->free_undo_status_node(undo_node); + } } return ret; } @@ -543,15 +630,15 @@ int ObTxData::merge_undo_actions_(ObTxDataTable *tx_data_table, // all undo actions in this node are merged, free it // STORAGE_LOG(DEBUG, "current node is empty, now free it"); ObUndoStatusNode *node_to_free = node; - undo_status_list_.head_ = node->next_; - node = undo_status_list_.head_; + op_guard_->get_undo_status_list().head_ = node->next_; + node = op_guard_->get_undo_status_list().head_; tx_data_table->free_undo_status_node(node_to_free); - if (undo_status_list_.undo_node_cnt_ <= 0) { + if (op_guard_->get_undo_status_list().undo_node_cnt_ <= 0) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(ERROR, "invalid undo node count int undo status list.", KR(ret), - K(undo_status_list_)); + K(op_guard_->get_undo_status_list())); } else { - undo_status_list_.undo_node_cnt_--; + op_guard_->get_undo_status_list().undo_node_cnt_--; } } else { // merge undo actions done @@ -580,12 +667,15 @@ bool ObTxData::equals_(ObTxData &rhs) } else if (end_scn_ != rhs.end_scn_) { bool_ret = false; STORAGE_LOG(INFO, "end_scn is not equal."); - } else if (undo_status_list_.undo_node_cnt_ != rhs.undo_status_list_.undo_node_cnt_) { - bool_ret = false; - STORAGE_LOG(INFO, "undo_node_cnt is not equal."); } else { - ObUndoStatusNode *l_node = undo_status_list_.head_; - ObUndoStatusNode *r_node = rhs.undo_status_list_.head_; + ObUndoStatusNode *l_node = NULL; + if (op_guard_.is_valid()) { + l_node = op_guard_->get_undo_status_list().head_; + } + ObUndoStatusNode *r_node = NULL; + if (rhs.op_guard_.is_valid()) { + r_node = rhs.op_guard_->get_undo_status_list().head_; + } while ((nullptr != l_node) && (nullptr != r_node)) { if (l_node->size_ != r_node->size_) { @@ -632,7 +722,9 @@ void ObTxData::print_to_stderr(const ObTxData &tx_data) to_cstring(tx_data.commit_version_), get_state_string(tx_data.state_)); - tx_data.undo_status_list_.dump_2_text(stderr); + if (tx_data.op_guard_.is_valid()) { + tx_data.op_guard_->get_undo_status_list().dump_2_text(stderr); + } } void ObTxData::dump_2_text(FILE *fd) const @@ -650,7 +742,9 @@ void ObTxData::dump_2_text(FILE *fd) const to_cstring(commit_version_), get_state_string(state_)); - undo_status_list_.dump_2_text(fd); + if (op_guard_.is_valid()) { + op_guard_->get_undo_status_list().dump_2_text(fd); + } fprintf(fd, "\n}\n"); } @@ -665,7 +759,7 @@ DEF_TO_STRING(ObTxData) K_(commit_version), K_(start_scn), K_(end_scn), - K_(undo_status_list)); + K_(op_guard)); J_OBJ_END(); return pos; } @@ -679,7 +773,65 @@ DEF_TO_STRING(ObUndoStatusNode) return pos; } +int ObTxData::init_tx_op() +{ + int ret = OB_SUCCESS; + void *ptr = nullptr; + if (!op_guard_.is_valid()) { + if (OB_ISNULL(tx_data_allocator_)) { + tx_data_allocator_ = &MTL(ObSharedMemAllocMgr*)->tx_data_allocator(); + } + if (OB_ISNULL(op_allocator_)) { + op_allocator_ = &MTL(ObSharedMemAllocMgr*)->tx_data_op_allocator(); + } + if (OB_ISNULL(ptr = tx_data_allocator_->alloc())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "allocate memory from slice_allocator fail.", KR(ret), KP(this)); + } else { + ObTxDataOp *tx_data_op = new (ptr) ObTxDataOp(tx_data_allocator_, op_allocator_); + op_guard_.init(tx_data_op); + } + } + return ret; +} + +int ObTxData::check_tx_op_exist(share::SCN op_scn, bool &exist) +{ + int ret = OB_SUCCESS; + exist = false; + if (op_guard_.is_valid()) { + ObTxOpVector &tx_op_list = op_guard_->get_tx_op_list(); + if (tx_op_list.get_count() > 0 && op_scn <= tx_op_list.at(tx_op_list.get_count() - 1)->get_op_scn()) { + exist = true; + } + } + return ret; +} + +int ObTxDataOpGuard::init(ObTxDataOp *tx_data_op) +{ + int ret = OB_SUCCESS; + reset(); + if (OB_ISNULL(tx_data_op)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "init ObTxDataOpGuard with invalid arguments", KR(ret)); + } else if (tx_data_op->inc_ref() <= 0) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(ERROR, "unexpected ref cnt on tx data op", KR(ret), KP(tx_data_op), KPC(tx_data_op)); + ob_abort(); + } else { + tx_data_op_ = tx_data_op; + } + return ret; +} + +void ObTxDataOpGuard::reset() +{ + if (OB_NOT_NULL(tx_data_op_)) { + tx_data_op_->dec_ref(); + tx_data_op_ = nullptr; + } +} } // namespace storage - } // namespace oceanbase diff --git a/src/storage/tx/ob_tx_data_define.h b/src/storage/tx/ob_tx_data_define.h index 0ceefb22ab..115a6c71cf 100644 --- a/src/storage/tx/ob_tx_data_define.h +++ b/src/storage/tx/ob_tx_data_define.h @@ -19,10 +19,10 @@ #include "storage/tx/ob_committer_define.h" #include "storage/tx/ob_trans_define.h" #include "storage/tx_table/ob_tx_data_hash_map.h" +#include "storage/tx/ob_trans_factory.h" namespace oceanbase { - namespace storage { class ObTxData; @@ -30,6 +30,8 @@ class ObTxTable; class ObTxDataTable; class ObTxDataMemtable; class ObTxDataMemtableMgr; +class ObTxDataOp; + // The memory structures associated with tx data are shown below. They are designed for several // reasons: @@ -132,7 +134,6 @@ struct ObTxDataLinkNode TO_STRING_KV(KP_(next)); }; - struct ObUndoStatusList { private: @@ -221,7 +222,6 @@ public: share::SCN end_scn_; }; - class ObTxDataLink { public: @@ -232,6 +232,29 @@ public: ObTxDataLinkNode hash_node_; }; +class ObTxDataOpGuard +{ +public: + ObTxDataOpGuard() : tx_data_op_(nullptr) {} + ~ObTxDataOpGuard() { reset(); } + int init(ObTxDataOp *tx_data_op); + bool is_valid() const { return tx_data_op_ != nullptr; } + void reset(); + ObTxDataOp *ptr() const { return tx_data_op_; } + ObTxDataOp &operator*() { + return *tx_data_op_; + } + ObTxDataOp* operator->() { + return tx_data_op_; + } + ObTxDataOp* operator->() const { + return tx_data_op_; + } + TO_STRING_KV(KP(tx_data_op_)); +private: + ObTxDataOp *tx_data_op_; +}; + // DONT : Modify this definition class ObTxData : public ObTxCommitData, public ObTxDataLink { @@ -248,8 +271,8 @@ public: : ObTxCommitData(), ObTxDataLink(), tx_data_allocator_(nullptr), + op_allocator_(nullptr), ref_cnt_(0), - undo_status_list_(), exclusive_flag_(ExclusiveType::NORMAL) {} ObTxData(const ObTxData &rhs); ObTxData &operator=(const ObTxData &rhs); @@ -260,6 +283,8 @@ public: void reset(); OB_INLINE bool contain(const transaction::ObTransID &tx_id) { return tx_id_ == tx_id; } + int init_tx_op(); + int reserve_undo(ObTxTable *tx_table); int64_t inc_ref() { int64_t ref_cnt = ATOMIC_AAF(&ref_cnt_, 1); @@ -275,23 +300,17 @@ public: STORAGE_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "invalid slice allocator", KPC(this)); ob_abort(); } else if (0 == ATOMIC_SAF(&ref_cnt_, 1)) { - if (OB_UNLIKELY(nullptr != undo_status_list_.head_)) { - ObUndoStatusNode *node_ptr = undo_status_list_.head_; - ObUndoStatusNode *node_to_free = nullptr; - while (nullptr != node_ptr) { - node_to_free = node_ptr; - node_ptr = node_ptr->next_; - tx_data_allocator_->free(node_to_free); - } - } + op_guard_.reset(); tx_data_allocator_->free(this); } } + int check_tx_op_exist(share::SCN op_scn, bool &exist); + /** * @brief Add a undo action with dynamically memory allocation. * See more details in alloc_undo_status_node() function of class ObTxDataTable - * + * * @param[in] tx_table, the tx table contains this tx data * @param[in & out] undo_action, the undo action which is waiting to be added. If this undo action contains exsiting undo actions, the existing undo actions will be deleted and this undo action will be modified to contain all the deleted undo actions. * @param[in] undo_node, the undo status node can be used to extend undo status list if required, otherwise it will be released @@ -306,7 +325,7 @@ public: int serialize(char *buf, const int64_t buf_len, int64_t &pos) const; int deserialize(const char *buf, const int64_t data_len, int64_t &pos, share::ObTenantTxDataAllocator &tx_data_allocator); int64_t get_serialize_size() const; - int64_t size() const; + int64_t size_need_cache() const; void dump_2_text(FILE *fd) const; static void print_to_stderr(const ObTxData &tx_data); @@ -338,11 +357,14 @@ public: public: share::ObTenantTxDataAllocator *tx_data_allocator_; + share::ObTenantTxDataOpAllocator *op_allocator_; int64_t ref_cnt_; - ObUndoStatusList undo_status_list_; ExclusiveType exclusive_flag_; + ObTxDataOpGuard op_guard_; }; +static_assert(sizeof(ObTxData) < storage::TX_DATA_SLICE_SIZE, "ObTxData exceed slice_allocator fixed length"); + class ObTxDataGuard { public: @@ -460,15 +482,19 @@ struct ObReadTxDataArg{ const transaction::ObTransID tx_id_; const int64_t read_epoch_; ObTxDataMiniCache &tx_data_mini_cache_; + const bool skip_cache_; - ObReadTxDataArg(const transaction::ObTransID tx_id, const int64_t read_epoch, ObTxDataMiniCache &mini_cache) - : tx_id_(tx_id), read_epoch_(read_epoch), tx_data_mini_cache_(mini_cache) {} + ObReadTxDataArg(const transaction::ObTransID tx_id, + const int64_t read_epoch, + ObTxDataMiniCache &mini_cache, + const bool skip_cache = false) + : tx_id_(tx_id), read_epoch_(read_epoch), + tx_data_mini_cache_(mini_cache), skip_cache_(skip_cache) {} - TO_STRING_KV(K_(tx_id), K_(read_epoch), K_(tx_data_mini_cache)); + TO_STRING_KV(K_(tx_id), K_(read_epoch), K_(tx_data_mini_cache), K_(skip_cache)); }; } // namespace storage - } // namespace oceanbase #endif // OCEANBASE_STORAGE_OB_TX_DATA_DEFINE_ diff --git a/src/storage/tx/ob_tx_data_functor.cpp b/src/storage/tx/ob_tx_data_functor.cpp index 6dbd5a1a25..95631e41d8 100644 --- a/src/storage/tx/ob_tx_data_functor.cpp +++ b/src/storage/tx/ob_tx_data_functor.cpp @@ -82,7 +82,8 @@ int CheckSqlSequenceCanReadFunctor::operator() (const ObTxData &tx_data, ObTxCCC const int32_t state = ATOMIC_LOAD(&tx_data.state_); const SCN commit_version = tx_data.commit_version_.atomic_load(); const SCN end_scn = tx_data.end_scn_.atomic_load(); - const bool is_rollback = tx_data.undo_status_list_.is_contain(sql_sequence_, state); + const bool is_rollback = !tx_data.op_guard_.is_valid() ? false : + tx_data.op_guard_->get_undo_status_list().is_contain(sql_sequence_, state); // NB: The functor is only used during minor merge if (ObTxData::ABORT == state) { @@ -116,7 +117,8 @@ int CheckRowLockedFunctor::operator() (const ObTxData &tx_data, ObTxCCCtx *tx_cc const int32_t state = ATOMIC_LOAD(&tx_data.state_); const SCN commit_version = tx_data.commit_version_.atomic_load(); const SCN end_scn = tx_data.end_scn_.atomic_load(); - const bool is_rollback = tx_data.undo_status_list_.is_contain(sql_sequence_, state); + const bool is_rollback = !tx_data.op_guard_.is_valid() ? false : + tx_data.op_guard_->get_undo_status_list().is_contain(sql_sequence_, state); switch (state) { case ObTxData::COMMIT: { @@ -249,7 +251,8 @@ int LockForReadFunctor::inner_lock_for_read(const ObTxData &tx_data, ObTxCCCtx * const int32_t state = ATOMIC_LOAD(&tx_data.state_); const SCN commit_version = tx_data.commit_version_.atomic_load(); const SCN end_scn = tx_data.end_scn_.atomic_load(); - const bool is_rollback = tx_data.undo_status_list_.is_contain(data_sql_sequence, state); + const bool is_rollback = !tx_data.op_guard_.is_valid() ? false : + tx_data.op_guard_->get_undo_status_list().is_contain(data_sql_sequence, state); can_read_ = false; trans_version_.set_invalid(); @@ -266,7 +269,7 @@ int LockForReadFunctor::inner_lock_for_read(const ObTxData &tx_data, ObTxCCCtx * } else { // Case 1.2: Otherwise, we get the version under mvcc can_read_ = snapshot_version >= commit_version - && !tx_data.undo_status_list_.is_contain(data_sql_sequence, state); + && !is_rollback; trans_version_ = commit_version; } break; @@ -500,7 +503,8 @@ int CleanoutTxStateFunctor::operator()(const ObTxData &tx_data, ObTxCCCtx *tx_cc const int32_t state = ATOMIC_LOAD(&tx_data.state_); const SCN commit_version = tx_data.commit_version_.atomic_load(); const SCN end_scn = tx_data.end_scn_.atomic_load(); - const bool is_rollback = tx_data.undo_status_list_.is_contain(seq_no_, state); + const bool is_rollback = !tx_data.op_guard_.is_valid() ? false : + tx_data.op_guard_->get_undo_status_list().is_contain(seq_no_, state); (void)resolve_tx_data_check_data_(state, commit_version, end_scn, is_rollback); @@ -632,9 +636,26 @@ int GenerateVirtualTxDataRowFunctor::operator()(const ObTxData &tx_data, ObTxCCC row_data_.start_scn_ = tx_data.start_scn_; row_data_.end_scn_ = tx_data.end_scn_; row_data_.commit_version_ = tx_data.commit_version_; - tx_data.undo_status_list_.to_string(row_data_.undo_status_list_str_, common::MAX_UNDO_LIST_CHAR_LENGTH); + if (tx_data.op_guard_.is_valid()) { + tx_data.op_guard_->get_undo_status_list().to_string(row_data_.undo_status_list_str_, common::MAX_UNDO_LIST_CHAR_LENGTH); + tx_data.op_guard_->get_tx_op_list().to_string(row_data_.tx_op_str_, common::MAX_TX_OP_CHAR_LENGTH); + } return OB_SUCCESS; } + +int LoadTxOpFunctor::operator()(const ObTxData &tx_data, ObTxCCCtx *tx_cc_ctx) +{ + int ret = OB_SUCCESS; + if (!tx_data.op_guard_.is_valid()) { + // do nothing + } else if (OB_FAIL(tx_data_.init_tx_op())) { + TRANS_LOG(WARN, "init_tx_op failed", K(ret)); + } else { + tx_data_.op_guard_.init(tx_data.op_guard_.ptr()); + } + return ret; +} + } // namespace storage } // namespace oceanbase diff --git a/src/storage/tx/ob_tx_data_functor.h b/src/storage/tx/ob_tx_data_functor.h index 0c6c15f16d..d1b87de150 100644 --- a/src/storage/tx/ob_tx_data_functor.h +++ b/src/storage/tx/ob_tx_data_functor.h @@ -241,6 +241,15 @@ public: observer::VirtualTxDataRow &row_data_; }; +class LoadTxOpFunctor : public ObITxDataCheckFunctor +{ +public: + LoadTxOpFunctor(ObTxData &tx_data) : tx_data_(tx_data) {} + virtual int operator()(const ObTxData &tx_data, ObTxCCCtx *tx_cc_ctx = nullptr) override; +public: + ObTxData &tx_data_; +}; + } // namespace storage } // namespace oceanbase diff --git a/src/storage/tx/ob_tx_data_op.cpp b/src/storage/tx/ob_tx_data_op.cpp new file mode 100644 index 0000000000..a434cfdc48 --- /dev/null +++ b/src/storage/tx/ob_tx_data_op.cpp @@ -0,0 +1,337 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "storage/tx/ob_tx_data_define.h" +#include "lib/utility/ob_unify_serialize.h" +#include "storage/tx_table/ob_tx_table.h" +#include "share/rc/ob_tenant_base.h" +#include "share/allocator/ob_shared_memory_allocator_mgr.h" + +using namespace oceanbase::share; +using namespace oceanbase::transaction; + +namespace oceanbase +{ +namespace storage +{ + +OB_SERIALIZE_MEMBER(ObTxDummyOp); +ObTxDummyOp DEFAULT_TX_DUMMY_OP; + +void ObTxDataOp::dec_ref() { + int ret = OB_SUCCESS; + if(ATOMIC_SAF(&ref_cnt_, 1) == 0) { + if (OB_FAIL(tx_op_list_.check_stat())) { + STORAGE_LOG(WARN, "dec_ref", KR(ret), KP(this), KPC(this)); + abort(); + } + // to release tx_op + for (int64_t idx = 0; idx < tx_op_list_.get_count(); idx++) { + ObTxOp &tx_op = *tx_op_list_.at(idx); + tx_op.release(); + } + if (OB_NOT_NULL(tx_op_list_.get_ptr())) { + op_allocator_->free(tx_op_list_.get_ptr()); + } + // to release undo status + if (OB_UNLIKELY(nullptr != undo_status_list_.head_)) { + ObUndoStatusNode *node_ptr = undo_status_list_.head_; + ObUndoStatusNode *node_to_free = nullptr; + while (nullptr != node_ptr) { + node_to_free = node_ptr; + node_ptr = node_ptr->next_; + tx_data_allocator_->free(node_to_free); + } + } + tx_data_allocator_->free(this); + } +} + +int64_t ObTxDataOp::get_tx_op_size() +{ + int64_t tx_op_size = tx_op_list_.get_capacity() * sizeof(ObTxOp); + for (int64_t idx = 0; idx < tx_op_list_.get_count(); idx++) { + ObTxOp &tx_op = *tx_op_list_.at(idx); + tx_op_size += tx_op.get_val_size(); + } + return tx_op_size; +} + +int64_t ObTxOpVector::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + + common::databuff_printf(buf, buf_len, pos, "count=%ld", count_); + for (int64_t idx = 0; idx < count_; idx++) { + ObTxOp *op = &tx_op_[idx]; + common::databuff_printf(buf, buf_len, pos, " op(%ld)=(op_code:%ld, op_scn:%ld)", + idx, op->get_op_code(), op->get_op_scn().convert_to_ts()); + } + return pos; +} + +ObTxOp *ObTxOpVector::at(int64_t idx) +{ + ObTxOp *tx_op = nullptr; + int ret = OB_SUCCESS; + if (OB_FAIL(check_stat())) { + STORAGE_LOG(WARN, "tx_op vector stat error", KPC(this), KR(ret), K(lbt())); + } else if (idx >= 0 && idx < count_) { + tx_op = &tx_op_[idx]; + } else { + STORAGE_LOG(WARN, "out of range", KPC(this), K(idx)); + } + return tx_op; +} + +int ObTxOpVector::push_back(ObTxOp &tx_op) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(check_stat())) { + STORAGE_LOG(WARN, "tx_op vector stat error", KR(ret)); + } else if (count_ < capacity_) { + tx_op_[count_] = tx_op; + count_++; + } else { + ret = OB_SIZE_OVERFLOW; + } + return ret; +} + +int ObTxOpVector::check_stat() +{ + int ret = OB_SUCCESS; + if (count_ < 0 || capacity_ < 0 || count_ > capacity_ + || (count_ > 0 && OB_ISNULL(tx_op_))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "check_stat failed", KPC(this), KP(this)); + } + return ret; +} + +int ObTxOpVector::try_extend_space(int64_t count, ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + if (count < 0) { + ret = OB_INVALID_ARGUMENT; + } else if (OB_FAIL(check_stat())) { + STORAGE_LOG(WARN, "check_stat failed", KR(ret)); + } else if (count == 0) { + // do nothing + } else if (count_ + count <= capacity_) { + // do nothing + } else { + ObTxOp *tx_op_ptr = nullptr; + if (OB_ISNULL(tx_op_ptr = (ObTxOp*)allocator.alloc((count_ + count) * sizeof(ObTxOp)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + } else { + if (count_ > 0) { + MEMCPY(tx_op_ptr, tx_op_, count_ * sizeof(ObTxOp)); + } + if (OB_NOT_NULL(tx_op_)) { + allocator.free(tx_op_); + } + tx_op_ = tx_op_ptr; + capacity_ = count_ + count; + } + } + return ret; +} + +int64_t ObTxOpVector::get_serialize_size() const +{ + int64_t len = 0; + len += serialization::encoded_length_vi64(count_); + for (int64_t idx = 0; idx < count_; idx++) { + len += tx_op_[idx].get_serialize_size(); + } + return len; +} + +int ObTxOpVector::serialize(char *buf, const int64_t buf_len, int64_t &pos) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(serialization::encode_vi64(buf, buf_len, pos, count_))) { + STORAGE_LOG(WARN, "serialize fail", KR(ret)); + } else { + for (int64_t idx = 0; OB_SUCC(ret) && idx < count_; idx++) { + if (OB_FAIL(tx_op_[idx].serialize(buf, buf_len, pos))) { + STORAGE_LOG(WARN, "serialize fail", KR(ret)); + } + } + } + return ret; +} + +int ObTxOpVector::deserialize(const char *buf, const int64_t buf_len, int64_t &pos, ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(serialization::decode_vi64(buf, buf_len, pos, &count_))) { + STORAGE_LOG(WARN, "deserialize fail", KR(ret), K(buf), K(buf_len), K(pos), K(count_)); + } else if (count_ > 0) { + if (OB_ISNULL(tx_op_ = (ObTxOp*)allocator.alloc(count_ * sizeof(ObTxOp)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "alloc mem fail", KR(ret), K(count_)); + count_ = 0; // reset count_ make vector clear + } else { + capacity_ = count_; + } + for (int64_t idx = 0; OB_SUCC(ret) && idx < count_; idx++) { + new (&tx_op_[idx]) ObTxOp(); + } + for (int64_t idx = 0; OB_SUCC(ret) && idx < count_; idx++) { + if (OB_FAIL(tx_op_[idx].deserialize(buf, buf_len, pos, allocator))) { + STORAGE_LOG(WARN, "deserialize fail", KR(ret)); + } + } + } + return ret; +} + +int ObTxDataOp::add_tx_op(ObTxOp &tx_op) +{ + int ret = OB_SUCCESS; + SpinWLockGuard lock_guard(lock_); + if (OB_FAIL(tx_op_list_.try_extend_space(1, *op_allocator_))) { + STORAGE_LOG(WARN, "try_extend_space fail", KR(ret), K(tx_op)); + } else if (OB_FAIL(tx_op_list_.push_back(tx_op))) { + STORAGE_LOG(WARN, "push tx_op to array failed", KR(ret)); + } + return ret; +} + +int ObTxDataOp::reserve_tx_op_space(int64_t count) +{ + int ret = OB_SUCCESS; + SpinWLockGuard lock_guard(lock_); + if (OB_FAIL(tx_op_list_.try_extend_space(count, *op_allocator_))) { + STORAGE_LOG(WARN, "try_extend_space fail", KR(ret), K(count)); + } + return ret; +} + +int ObTxDataOp::add_tx_op_batch(transaction::ObTransID tx_id, share::ObLSID ls_id, share::SCN op_scn, ObTxOpArray &tx_op_batch) +{ + int ret = OB_SUCCESS; + SpinWLockGuard lock_guard(lock_); + if (OB_FAIL(tx_op_list_.try_extend_space(tx_op_batch.count(), *op_allocator_))) { + STORAGE_LOG(WARN, "try_extend_space fail", KR(ret), K(tx_op_batch)); + } else { + for (int64_t idx = 0; OB_SUCC(ret) && idx < tx_op_batch.count(); idx++) { + if (OB_FAIL(tx_op_list_.push_back(tx_op_batch.at(idx)))) { + STORAGE_LOG(WARN, "push tx_op to array failed", KR(ret)); + } + } + // !!! we must promise tx_op_batch atomic append into tx_op_list + // otherwise tx_op replay filter with log_scn compare op_scn will cause serious problem + if (OB_FAIL(ret)) { + STORAGE_LOG(ERROR, "tx_op_batch is not atomic append", K(tx_id), K(ls_id), K(tx_op_list_), K(tx_op_batch), K(op_scn)); + ob_abort(); + } + } + STORAGE_LOG(INFO, "add_tx_op", K(ret), K(tx_id), K(ls_id), K(op_scn), K(tx_op_batch.count()), K(tx_op_list_.get_count()), K(tx_op_batch)); + return ret; +} + +int64_t ObTxOp::get_serialize_size() const +{ + int64_t len = 0; + len += serialization::encoded_length_vi64(int64_t(op_code_)); + len += op_scn_.get_serialize_size(); + + #define SERIALIZE_TX_OP_TMP(OP_CODE, OP_TYPE) \ + if (OB_NOT_NULL(op_val_) && op_code_ == OP_CODE) { \ + OP_TYPE &op_obj = *((OP_TYPE*)op_val_); \ + len += op_obj.get_serialize_size(); \ + } + #define SERIALIZE_TX_OP(TYPE, UNUSED) SERIALIZE_TX_OP_TMP TYPE + + LST_DO2(SERIALIZE_TX_OP, (), TX_OP_MEMBERS); + #undef SERIALIZE_TX_OP_TMP + #undef SERIALIZE_TX_OP + return len; +} + +int ObTxOp::serialize(char *buf, const int64_t buf_len, int64_t &pos) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(serialization::encode_vi64(buf, buf_len, pos, int64_t(op_code_)))) { + STORAGE_LOG(WARN, "serialize fail", K(ret), K(buf_len), K(pos)); + } else if (OB_FAIL(op_scn_.serialize(buf, buf_len, pos))) { + STORAGE_LOG(WARN, "serialize fail", K(ret), K(buf_len), K(pos)); + } else if (OB_ISNULL(op_val_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "tx_op op_val is null", KR(ret), KPC(this)); + } + #define SERIALIZE_TX_OP_TMP(OP_CODE, OP_TYPE) \ + if (OB_SUCC(ret) && op_code_ == OP_CODE) { \ + OP_TYPE &op_obj = *((OP_TYPE*)op_val_); \ + if (OB_FAIL(op_obj.serialize(buf, buf_len, pos))) { \ + STORAGE_LOG(WARN, "serialize fail", KR(ret)); \ + } \ + } + #define SERIALIZE_TX_OP(TYPE, UNUSED) SERIALIZE_TX_OP_TMP TYPE + + LST_DO2(SERIALIZE_TX_OP, (), TX_OP_MEMBERS); + #undef SERIALIZE_TX_OP_TMP + #undef SERIALIZE_TX_OP + return ret; +} + +int ObTxOp::deserialize(const char *buf, const int64_t data_len, int64_t &pos, ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(serialization::decode_vi64(buf, data_len, pos, (int64_t*)&op_code_))) { + STORAGE_LOG(WARN, "deserialize fail", K(data_len), K(pos), K(ret)); + } else if (OB_FAIL(op_scn_.deserialize(buf, data_len, pos))) { + STORAGE_LOG(WARN, "deserialize fail", K(ret), K(data_len), K(pos)); + } + #define DESERIALIZE_TX_OP_TMP(OP_CODE, OP_TYPE) \ + if (OB_SUCC(ret) && op_code_ == OP_CODE) { \ + OP_TYPE *op_obj = nullptr; \ + if (OB_ISNULL(op_obj = (OP_TYPE*)allocator.alloc(sizeof(OP_TYPE)))) { \ + ret = OB_ALLOCATE_MEMORY_FAILED; \ + STORAGE_LOG(WARN, "deserialize fail", K(ret), K(data_len), K(pos)); \ + } else if (FALSE_IT(new (op_obj) OP_TYPE())) { \ + } else if (OB_FAIL(op_obj->deserialize(buf, data_len, pos))) { \ + STORAGE_LOG(WARN, "deserialize fail", KR(ret)); \ + allocator.free(op_obj); \ + } else if (FALSE_IT(op_val_ = op_obj)) { \ + } \ + } + #define DESERIALIZE_TX_OP(TYPE, UNUSED) DESERIALIZE_TX_OP_TMP TYPE + + LST_DO2(DESERIALIZE_TX_OP, (), TX_OP_MEMBERS); + #undef DESERIALIZE_TX_OP_TMP + #undef DESERIALIZE_TX_OP + return ret; +} + +void ObTxOp::release() +{ + ObIAllocator &allocator = MTL(ObSharedMemAllocMgr*)->tx_data_op_allocator(); + #define RELEASE_TX_OP_TMP(OP_CODE, OP_TYPE) \ + if (OB_NOT_NULL(op_val_) && op_code_ == OP_CODE \ + && op_val_ != &DEFAULT_TX_DUMMY_OP) { \ + OP_TYPE *op_obj = (OP_TYPE*)op_val_; \ + release(*op_obj); \ + allocator.free(op_obj); \ + } + #define RELEASE_TX_OP(TYPE, UNUSED) RELEASE_TX_OP_TMP TYPE + + LST_DO2(RELEASE_TX_OP, (), TX_OP_MEMBERS); + #undef RELEASE_TX_OP_TMP + #undef RELEASE_TX_OP +} + +} // namespace storage +} // namespace oceanbase diff --git a/src/storage/tx/ob_tx_data_op.h b/src/storage/tx/ob_tx_data_op.h new file mode 100644 index 0000000000..cc5cfb2b0f --- /dev/null +++ b/src/storage/tx/ob_tx_data_op.h @@ -0,0 +1,213 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_STORAGE_OB_TX_DATA_OP +#define OCEANBASE_STORAGE_OB_TX_DATA_OP + +#include "share/scn.h" +#include "share/allocator/ob_tx_data_allocator.h" +#include "storage/tx/ob_trans_define.h" +#include "storage/tx/ob_tx_data_define.h" + +namespace oceanbase +{ + +namespace transaction +{ +namespace tablelock +{ + struct ObTableLockOp; +} +} +namespace storage +{ + +// record tx.op has no value +class ObTxDummyOp +{ + OB_UNIS_VERSION(1); +public: + TO_STRING_EMPTY(); +}; + +extern ObTxDummyOp DEFAULT_TX_DUMMY_OP; + +/* + * tx.op is designed to store tx operations that need retain after tx_ctx exit + * so tx.op need to support various data types + * + * we use class ObTxOp to describe every tx operation + * users just need to add new data types to TX_OP_MEMBERS, we can create ObTxOp and put it into ObTxDataOp + */ +enum class ObTxOpCode : int64_t +{ + INVALID = 0, + MDS_OP = 1, + LOCK_OP = 2, + ABORT_OP = 3 +}; + +#define TX_OP_LIST(...) __VA_ARGS__ +#define TX_OP_MEMBERS \ + TX_OP_LIST( \ + (ObTxOpCode::MDS_OP, transaction::ObTxBufferNodeWrapper), \ + (ObTxOpCode::LOCK_OP, transaction::tablelock::ObTableLockOp), \ + (ObTxOpCode::ABORT_OP, ObTxDummyOp) \ + ) + +class ObTxOp +{ +public: + ObTxOp() { reset(); } + ~ObTxOp() { reset(); } + void reset() { + op_code_ = ObTxOpCode::INVALID; + op_scn_.reset(); + op_val_ = nullptr; + val_size_ = 0; + } + template + int init(ObTxOpCode op_code, share::SCN op_scn, T *val, int64_t val_size); + ObTxOpCode get_op_code() { return op_code_; } + share::SCN get_op_scn() { return op_scn_; } + void set_op_scn(share::SCN scn) { op_scn_ = scn; } + void* get_op_val() { return op_val_; } + int64_t get_val_size() { return val_size_; } + template + T *get(); + int64_t get_serialize_size() const; + int serialize(char *buf, const int64_t buf_len, int64_t &pos) const; + int deserialize(const char *buf, const int64_t data_len, int64_t &pos, ObIAllocator &allocator); + void release(); + template + void release(T &op_val) { + op_val.~T(); + } + TO_STRING_KV(K_(op_code), K_(op_scn), K_(op_val)); +private: + ObTxOpCode op_code_; + share::SCN op_scn_; + void *op_val_; + int64_t val_size_; // for tx_data memory +}; + +typedef ObSEArray ObTxOpArray; + +class ObTxOpVector +{ +public: + ObTxOpVector() { reset(); } + ~ObTxOpVector() { reset(); } + void reset() { + capacity_ = 0; + count_ = 0; + tx_op_ = nullptr; + } + int64_t get_capacity() { return capacity_; } + int64_t get_count() { return count_; } + ObTxOp* get_ptr() { return tx_op_; } + int try_extend_space(int64_t count, ObIAllocator &allocator); + ObTxOp *at(int64_t idx); + int push_back(ObTxOp &tx_op); + int serialize(char *buf, const int64_t buf_len, int64_t &pos) const; + int deserialize(const char *buf, const int64_t data_len, int64_t &pos, ObIAllocator &allocator); + int64_t get_serialize_size() const; + int check_stat(); + int64_t to_string(char *buf, const int64_t buf_len) const; +private: + int64_t capacity_; + int64_t count_; + ObTxOp *tx_op_; +}; + +class ObTxDataOp +{ +public: + ObTxDataOp(share::ObTenantTxDataAllocator *allocator, share::ObTenantTxDataOpAllocator *op_allocator) : + ref_cnt_(0), + undo_status_list_(), + tx_op_list_(), + tx_data_allocator_(allocator), + op_allocator_(op_allocator) {} + ~ObTxDataOp() {} + int64_t inc_ref() { return ATOMIC_AAF(&ref_cnt_, 1); } + void dec_ref(); + ObTxOpVector &get_tx_op_list() { return tx_op_list_; } + int64_t get_tx_op_size(); + ObUndoStatusList &get_undo_status_list() { return undo_status_list_; } + common::SpinRWLock &get_lock() { return lock_; } + int64_t get_ref() { return ref_cnt_; } + int add_tx_op(ObTxOp &tx_op); + int add_tx_op_batch(transaction::ObTransID tx_id, share::ObLSID ls_id, share::SCN op_scn, ObTxOpArray &tx_op_batch); + int reserve_tx_op_space(int64_t count); + + TO_STRING_KV(K_(ref_cnt), K_(undo_status_list), K_(tx_op_list)); +private: + int64_t ref_cnt_; + ObUndoStatusList undo_status_list_; + ObTxOpVector tx_op_list_; + share::ObTenantTxDataAllocator *tx_data_allocator_; + share::ObTenantTxDataOpAllocator *op_allocator_; + common::SpinRWLock lock_; +}; + +static_assert(sizeof(ObTxDataOp) < TX_DATA_SLICE_SIZE, "ObTxDataOp too large!"); + +template +int ObTxOp::init(ObTxOpCode op_code, share::SCN op_scn, T* op_val, int64_t val_size) +{ + int ret = OB_SUCCESS; + if (op_code == ObTxOpCode::INVALID || !op_scn.is_valid() || OB_ISNULL(op_val) || val_size < 0) { + ret = OB_INVALID_ARGUMENT; + } + + #define INIT_TX_OP_TMP(OP_CODE, OP_TYPE) \ + if (OB_SUCC(ret) && op_code_ == OP_CODE && !(std::is_same::value)) { \ + ret = OB_INVALID_ARGUMENT; \ + } + #define INIT_TX_OP(TYPE, UNUSED) INIT_TX_OP_TMP TYPE + + LST_DO2(INIT_TX_OP, (), TX_OP_MEMBERS); + #undef INIT_TX_OP_TMP + #undef INIT_TX_OP + if (OB_SUCC(ret)) { + op_code_ = op_code; + op_scn_ = op_scn; + op_val_ = op_val; + val_size_ = val_size; + } + return ret; +} + +template +T* ObTxOp::get() +{ + T* val = nullptr; + + #define GET_TX_OP_TMP(OP_CODE, OP_TYPE) \ + if (std::is_same::value) { \ + val = (T*)op_val_; \ + } + #define GET_TX_OP(TYPE, UNUSED) GET_TX_OP_TMP TYPE + + LST_DO2(GET_TX_OP, (), TX_OP_MEMBERS); + #undef GET_TX_OP_TMP + #undef GET_TX_OP + + return val; +} + + +} // namespace storage +} // namespace oceanbase + +#endif // OCEANBASE_STORAGE_OB_TX_DATA_OP_ diff --git a/src/storage/tx/ob_tx_log.h b/src/storage/tx/ob_tx_log.h index 58329987a6..35d8c850a8 100644 --- a/src/storage/tx/ob_tx_log.h +++ b/src/storage/tx/ob_tx_log.h @@ -411,11 +411,12 @@ public: is_alloc_ = true; dli_buf_size_ = ddl_log->get_serialize_size(); submit_buf_ = static_cast(share::mtl_malloc(dli_buf_size_, "DLI_TMP_BUF")); - memset(submit_buf_, 0, dli_buf_size_); int64_t pos = 0; if (OB_ISNULL(submit_buf_)) { ret = OB_ALLOCATE_MEMORY_FAILED; TRANS_LOG(WARN, "alloc memory failed", K(ret), KPC(this), KPC(ddl_log)); + } else if (OB_FALSE_IT(memset(submit_buf_, 0, dli_buf_size_))) { + // do nothing } else if (ddl_log->serialize(static_cast(submit_buf_), dli_buf_size_, pos)) { TRANS_LOG(WARN, "serialize ddl log buf failed", K(ret), KPC(this), KPC(ddl_log)); } @@ -1100,7 +1101,8 @@ class ObTxAbortLog public: ObTxAbortLog(ObTxAbortLogTempRef &temp_ref) - : multi_source_data_(temp_ref.multi_source_data_), tx_data_backup_() + : multi_source_data_(temp_ref.multi_source_data_), + tx_data_backup_() { before_serialize(); } diff --git a/src/storage/tx/ob_tx_loop_worker.cpp b/src/storage/tx/ob_tx_loop_worker.cpp index 64e82ebe4a..0e76e57317 100644 --- a/src/storage/tx/ob_tx_loop_worker.cpp +++ b/src/storage/tx/ob_tx_loop_worker.cpp @@ -80,8 +80,9 @@ void ObTxLoopWorker::destroy() void ObTxLoopWorker::reset() { - last_tx_gc_ts_ = false; + last_tx_gc_ts_ = 0; last_retain_ctx_gc_ts_ = 0; + last_check_start_working_retry_ts_ = 0; } void ObTxLoopWorker::run1() @@ -92,6 +93,7 @@ void ObTxLoopWorker::run1() lib::set_thread_name("TxLoopWorker"); bool can_gc_tx = false; bool can_gc_retain_ctx = false; + bool can_check_and_retry_start_working = false; while (!has_set_stop()) { start_time_us = ObTimeUtility::current_time(); @@ -113,7 +115,14 @@ void ObTxLoopWorker::run1() can_gc_retain_ctx = true; } - (void)scan_all_ls_(can_gc_tx, can_gc_retain_ctx); + if (common::ObClockGenerator::getClock() - last_check_start_working_retry_ts_ + > TX_START_WORKING_RETRY_INTERVAL) { + TRANS_LOG(INFO, "try to retry start_working"); + last_check_start_working_retry_ts_ = common::ObClockGenerator::getClock(); + can_check_and_retry_start_working = true; + } + + (void)scan_all_ls_(can_gc_tx, can_gc_retain_ctx, can_check_and_retry_start_working); // TODO shanyan.g // 1) We use max(max_commit_ts, gts_cache) as read snapshot, @@ -128,10 +137,13 @@ void ObTxLoopWorker::run1() } can_gc_tx = false; can_gc_retain_ctx = false; + can_check_and_retry_start_working = false; } } -int ObTxLoopWorker::scan_all_ls_(bool can_tx_gc, bool can_gc_retain_ctx) +int ObTxLoopWorker::scan_all_ls_(bool can_tx_gc, + bool can_gc_retain_ctx, + bool can_check_and_retry_start_working) { int ret = OB_SUCCESS; int iter_ret = OB_SUCCESS; @@ -164,7 +176,7 @@ int ObTxLoopWorker::scan_all_ls_(bool can_tx_gc, bool can_gc_retain_ctx) if (OB_TMP_FAIL(cur_ls_ptr->get_log_handler()->get_role(role, base_proposal_id))) { TRANS_LOG(WARN, "get role failed", K(tmp_ret), K(cur_ls_ptr->get_ls_id())); - status = MinStartScnStatus::UNKOWN; + status = MinStartScnStatus::UNKOWN; } else if (role == common::ObRole::FOLLOWER) { status = MinStartScnStatus::UNKOWN; } @@ -184,7 +196,7 @@ int ObTxLoopWorker::scan_all_ls_(bool can_tx_gc, bool can_gc_retain_ctx) do_tx_gc_(cur_ls_ptr, min_start_scn, status); } - if(MinStartScnStatus::UNKOWN == status) { + if (MinStartScnStatus::UNKOWN == status) { // do nothing } else if (OB_TMP_FAIL(cur_ls_ptr->get_log_handler()->get_role(role, proposal_id))) { TRANS_LOG(WARN, "get role failed", K(tmp_ret), K(cur_ls_ptr->get_ls_id())); @@ -222,6 +234,10 @@ int ObTxLoopWorker::scan_all_ls_(bool can_tx_gc, bool can_gc_retain_ctx) if (can_gc_retain_ctx) { do_retain_ctx_gc_(cur_ls_ptr); } + + if (can_check_and_retry_start_working) { + do_start_working_retry_(cur_ls_ptr); + } } } @@ -308,6 +324,15 @@ void ObTxLoopWorker::do_retain_ctx_gc_(ObLS *ls_ptr) UNUSED(ret); } +void ObTxLoopWorker::do_start_working_retry_(ObLS *ls_ptr) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(ls_ptr->retry_apply_start_working_log())) { + TRANS_LOG(WARN, "retry to apply start working log failed", K(ret), KPC(ls_ptr)); + } +} + } } diff --git a/src/storage/tx/ob_tx_loop_worker.h b/src/storage/tx/ob_tx_loop_worker.h index b6fdc93683..f4e8a68667 100644 --- a/src/storage/tx/ob_tx_loop_worker.h +++ b/src/storage/tx/ob_tx_loop_worker.h @@ -40,6 +40,7 @@ public: const static int64_t KEEP_ALIVE_PRINT_INFO_INTERVAL = 5 * 60 * 1000 * 1000; // 5min const static int64_t TX_GC_INTERVAL = 5 * 1000 * 1000; // 5s const static int64_t TX_RETAIN_CTX_GC_INTERVAL = 5 * 1000 * 1000; // 5s + const static int64_t TX_START_WORKING_RETRY_INTERVAL = 5 * 1000 * 1000; //5s public: ObTxLoopWorker() { reset(); } ~ObTxLoopWorker() {} @@ -55,15 +56,17 @@ public: virtual void run1(); private: - int scan_all_ls_(bool can_tx_gc, bool can_gc_retain_ctx); + int scan_all_ls_(bool can_tx_gc, bool can_gc_retain_ctx, bool can_check_and_retry_start_working); void do_keep_alive_(ObLS *ls, const share::SCN &min_start_scn, MinStartScnStatus status); // 100ms void do_tx_gc_(ObLS *ls, share::SCN &min_start_scn, MinStartScnStatus &status); // 15s void update_max_commit_ts_(); void do_retain_ctx_gc_(ObLS * ls); // 15s + void do_start_working_retry_(ObLS * ls); private: int64_t last_tx_gc_ts_; int64_t last_retain_ctx_gc_ts_; + int64_t last_check_start_working_retry_ts_; }; diff --git a/src/storage/tx/ob_tx_ls_log_writer.h b/src/storage/tx/ob_tx_ls_log_writer.h index bd486f5d64..c091435144 100644 --- a/src/storage/tx/ob_tx_ls_log_writer.h +++ b/src/storage/tx/ob_tx_ls_log_writer.h @@ -139,7 +139,7 @@ int ObTxLSLogCb::serialize_ls_log(T &ls_log, class ObTxLSLogWriter { public: - const static uint8_t DEFAULT_LOG_CB_CNT = 5; + const static uint8_t DEFAULT_LOG_CB_CNT = 1; const static uint8_t APPEND_LOG_CB_CNT = 1; typedef common::ObSEArray @@ -166,12 +166,6 @@ public: int on_success(ObTxLSLogCb *cb); int on_failure(ObTxLSLogCb *cb); - // TODO RoleChangeSubHandler - void switch_to_follower_forcedly(); - int switch_to_leader(); - int switch_to_follower_gracefully(); - int resume_leader(); - private: template int submit_ls_log_(T &ls_log, @@ -223,12 +217,12 @@ int ObTxLSLogWriter::submit_ls_log_(T &ls_log, } else if (nullptr == cb || OB_FAIL(cb->serialize_ls_log(ls_log, replay_hint, barrier_type))) { TRANS_LOG(WARN, "[TxLsLogWriter] serialize ls log error", KR(ret), K(T::LOG_TYPE), KP(cb)); } else if (OB_FAIL(tx_log_adapter_->submit_log(cb->get_log_buf(), cb->get_log_pos(), share::SCN::min_scn(), cb, need_nonblock))) { + TRANS_LOG(WARN, "[TxLsLogWriter] submit ls log failed", KR(ret), K(T::LOG_TYPE), K(ls_id_), KPC(cb)); return_log_cb_(cb); cb = nullptr; - TRANS_LOG(WARN, "[TxLsLogWriter] submit ls log failed", KR(ret), K(T::LOG_TYPE), K(ls_id_)); } else { log_ts = cb->get_log_ts(); - TRANS_LOG(INFO, "[TxLsLogWriter] submit ls log success", K(ret), K(T::LOG_TYPE), K(ls_id_)); + TRANS_LOG(INFO, "[TxLsLogWriter] submit ls log success", K(ret), K(T::LOG_TYPE), K(ls_id_), KPC(cb)); } return ret; } diff --git a/src/storage/tx/ob_tx_ls_state_mgr.cpp b/src/storage/tx/ob_tx_ls_state_mgr.cpp new file mode 100644 index 0000000000..50affbb86f --- /dev/null +++ b/src/storage/tx/ob_tx_ls_state_mgr.cpp @@ -0,0 +1,409 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "storage/tx/ob_tx_ls_state_mgr.h" +#include "storage/tx_storage/ob_ls_service.h" + +#define LOAD_CUR_STATE_CONTAINER \ + TxLSStateContainer ls_state_container; \ + ls_state_container.state_container_ = ATOMIC_LOAD(&cur_state_.state_container_); + +namespace oceanbase +{ + +namespace transaction +{ + +int ObTxLSStateMgr::TxLSStateHashFunc::operator()(const TxStateActionPair &key, + TxLSStateContainer &res) const +{ + int ret = OB_SUCCESS; + + res = key.base_state_; + switch (key.exec_action_) { + case TxLSAction::START: { + if (key.base_state_.state_val_.state_ == TxLSState::INIT) { + clear_all_flag_(res); + res.state_val_.state_ = TxLSState::F_WORKING; + } else { + ret = OB_STATE_NOT_MATCH; + } + break; + } + case TxLSAction::STOP: { + + if (key.base_state_.state_val_.state_ == TxLSState::INIT) { + // do nothing + } else { + clear_all_flag_(res); + res.state_val_.state_ = TxLSState::STOPPED; + } + break; + } + case TxLSAction::ONLINE: { + res.state_val_.offline_flag_ = false; + res.state_val_.block_start_tx_flag_ = false; + res.state_val_.block_start_readonly_flag_ = false; + break; + } + case TxLSAction::OFFLINE: { + res.state_val_.offline_flag_ = true; + break; + } + case TxLSAction::LEADER_REVOKE_GRACEFULLY: { + if (key.base_state_.state_val_.state_ == TxLSState::L_WORKING + || key.base_state_.state_val_.state_ == TxLSState::T_SYNC_FAILED + || key.base_state_.state_val_.state_ == TxLSState::R_SYNC_FAILED) { + res.state_val_.state_ = TxLSState::F_WORKING; + res.state_val_.block_start_normal_tx_flag_ = false; + } else { + ret = OB_STATE_NOT_MATCH; + } + break; + } + case TxLSAction::LEADER_REVOKE_FORCEDLLY: { + if (key.base_state_.state_val_.state_ == TxLSState::L_WORKING + || key.base_state_.state_val_.state_ == TxLSState::T_SYNC_FAILED + || key.base_state_.state_val_.state_ == TxLSState::R_SYNC_FAILED) { + res.state_val_.state_ = TxLSState::F_WORKING; + res.state_val_.block_start_normal_tx_flag_ = false; + } else if (key.base_state_.state_val_.state_ == TxLSState::T_APPLY_PENDING + || key.base_state_.state_val_.state_ == TxLSState::R_APPLY_PENDING) { + res.state_val_.state_ = TxLSState::F_SWL_PENDING; + res.state_val_.block_start_normal_tx_flag_ = false; + } else { + ret = OB_STATE_NOT_MATCH; + } + break; + } + case TxLSAction::SWL_CB_SUCC: { + if (key.base_state_.state_val_.state_ == TxLSState::T_SYNC_PENDING) { + res.state_val_.state_ = TxLSState::T_APPLY_PENDING; + } else if (key.base_state_.state_val_.state_ == TxLSState::R_SYNC_PENDING) { + res.state_val_.state_ = TxLSState::R_APPLY_PENDING; + } else { + ret = OB_STATE_NOT_MATCH; + } + break; + } + case TxLSAction::SWL_CB_FAIL: { + + if (key.base_state_.state_val_.state_ == TxLSState::T_SYNC_PENDING) { + res.state_val_.state_ = TxLSState::T_SYNC_FAILED; + } else if (key.base_state_.state_val_.state_ == TxLSState::R_SYNC_PENDING) { + res.state_val_.state_ = TxLSState::R_SYNC_FAILED; + } else { + ret = OB_STATE_NOT_MATCH; + } + break; + } + case TxLSAction::APPLY_SWL_SUCC: { + if (key.base_state_.state_val_.state_ == TxLSState::T_APPLY_PENDING + || key.base_state_.state_val_.state_ == TxLSState::R_APPLY_PENDING) { + res.state_val_.state_ = TxLSState::L_WORKING; + } else if (key.base_state_.state_val_.state_ == TxLSState::F_SWL_PENDING) { + res.state_val_.state_ = TxLSState::F_WORKING; + } else { + ret = OB_STATE_NOT_MATCH; + } + break; + } + case TxLSAction::APPLY_SWL_FAIL: { + if (key.base_state_.state_val_.state_ == TxLSState::T_APPLY_PENDING) { + res.state_val_.state_ = TxLSState::T_APPLY_PENDING; + } else if (key.base_state_.state_val_.state_ == TxLSState::R_APPLY_PENDING) { + res.state_val_.state_ = TxLSState::R_APPLY_PENDING; + } else if (key.base_state_.state_val_.state_ == TxLSState::F_SWL_PENDING) { + res.state_val_.state_ = TxLSState::F_SWL_PENDING; + } else { + ret = OB_STATE_NOT_MATCH; + } + break; + } + case TxLSAction::LEADER_TAKEOVER: { + if (key.base_state_.state_val_.state_ == TxLSState::F_WORKING) { + res.state_val_.state_ = TxLSState::T_SYNC_PENDING; + } else { + ret = OB_STATE_NOT_MATCH; + } + break; + } + case TxLSAction::RESUME_LEADER: { + if (key.base_state_.state_val_.state_ == TxLSState::F_WORKING) { + res.state_val_.state_ = TxLSState::R_SYNC_PENDING; + } else { + ret = OB_STATE_NOT_MATCH; + } + break; + } + case TxLSAction::BLOCK_START_TX: { + res.state_val_.block_start_tx_flag_ = true; + break; + } + case TxLSAction::BLOCK_START_READONLY: { + res.state_val_.block_start_readonly_flag_ = true; + break; + } + case TxLSAction::BLOCK_START_NORMAL_TX: { + if (key.base_state_.state_val_.state_ == TxLSState::L_WORKING) { + res.state_val_.block_start_normal_tx_flag_ = true; + } else { + ret = OB_STATE_NOT_MATCH; + } + break; + } + case TxLSAction::UNBLOCK_NORMAL_TX: { + if (key.base_state_.state_val_.state_ == TxLSState::L_WORKING) { + res.state_val_.block_start_normal_tx_flag_ = false; + } else { + ret = OB_STATE_NOT_MATCH; + } + break; + } + case TxLSAction::BLOCK_START_WR: { + res.state_val_.block_start_tx_flag_ = true; + res.state_val_.block_start_readonly_flag_ = true; + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(ERROR, "unkonwn action for ls_tx_state", K(ret), K(key), K(res.state_val_)); + } + } + + return ret; +} + +int ObTxLSStateMgr::switch_tx_ls_state(const TxLSAction action, const share::SCN start_working_scn) +{ + int ret = OB_SUCCESS; + if (TxLSAction::SWL_CB_SUCC == action) { + share::SCN tmp_max_applied_swl_scn = max_applied_start_working_ts_.atomic_load(); + if (!max_applying_start_working_ts_.atomic_bcas(tmp_max_applied_swl_scn, + tmp_max_applied_swl_scn)) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(ERROR, "prev start working log has not been applied", K(ret), "action", + action_str(action), K(start_working_scn), KPC(this)); + } else { + max_applying_start_working_ts_.inc_update(start_working_scn); + } + } else if (TxLSAction::APPLY_SWL_SUCC == action) { + if (!max_applying_start_working_ts_.atomic_bcas(start_working_scn, start_working_scn)) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(ERROR, "Try to applied a unexpected start working scn", K(ret), "action", + action_str(action), K(start_working_scn), KPC(this)); + } else { + max_applied_start_working_ts_.inc_update(start_working_scn); + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(execute_tx_ls_action_(action))) { + TRANS_LOG(WARN, "execute tx ls action failed", K(ret), K(ret), "action", action_str(action), + K(start_working_scn), KPC(this)); + } + return ret; +} + +void ObTxLSStateMgr::restore_tx_ls_state() +{ + int ret = OB_SUCCESS; + TxLSStateContainer restore_state_container; + restore_state_container.state_container_ = 0; + restore_state_container.state_container_ = ATOMIC_LOAD(&prev_state_.state_container_); + ATOMIC_STORE(&cur_state_.state_container_, restore_state_container.state_container_); + TRANS_LOG(INFO, "[Tx LS State] restore ls tx state successfully", K(ret), + K(restore_state_container.state_val_)); +} + +bool ObTxLSStateMgr::need_retry_apply_SWL(share::SCN &applying_swl_scn) +{ + bool need_retry = false; + LOAD_CUR_STATE_CONTAINER + if (ls_state_container.state_val_.state_ == TxLSState::R_APPLY_PENDING + || ls_state_container.state_val_.state_ == TxLSState::T_APPLY_PENDING + || ls_state_container.state_val_.state_ == TxLSState::F_SWL_PENDING) { + + share::SCN max_applied_swl_scn = max_applied_start_working_ts_.atomic_load(); + + if (max_applying_start_working_ts_.atomic_bcas(max_applied_swl_scn, max_applied_swl_scn)) { + need_retry = false; + } else { + applying_swl_scn = max_applying_start_working_ts_.atomic_load(); + need_retry = true; + } + } + + return need_retry; +} + +bool ObTxLSStateMgr::waiting_SWL_cb() +{ + bool is_waiting = false; + + LOAD_CUR_STATE_CONTAINER + + if (ls_state_container.state_val_.state_ == TxLSState::R_SYNC_PENDING + || ls_state_container.state_val_.state_ == TxLSState::T_SYNC_PENDING) { + is_waiting = true; + } + + return is_waiting; +} + +void ObTxLSStateMgr::replay_SWL_succ(const share::SCN &swl_scn) +{ + int ret = OB_SUCCESS; + share::SCN tmp_max_applied_swl_scn = max_applied_start_working_ts_.atomic_load(); + if (!max_applying_start_working_ts_.atomic_bcas(tmp_max_applied_swl_scn, + tmp_max_applied_swl_scn)) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(ERROR, "prev start working log has not been applied", K(ret), KPC(this)); + } + max_applying_start_working_ts_.inc_update(swl_scn); + max_applied_start_working_ts_.inc_update(swl_scn); +} + +const char *ObTxLSStateMgr::iter_ctx_mgr_stat_info(uint64_t &state_container, + bool &is_master, + bool &is_stopped) const +{ + LOAD_CUR_STATE_CONTAINER + is_master = (ls_state_container.state_val_.state_ == TxLSState::L_WORKING); + is_stopped = (ls_state_container.state_val_.state_ == TxLSState::STOPPED); + state_container = ls_state_container.state_container_; + return to_cstring(ls_state_container.state_val_); +} + +bool ObTxLSStateMgr::is_master() const +{ + LOAD_CUR_STATE_CONTAINER + return ls_state_container.state_val_.state_ == TxLSState::L_WORKING; +} + +bool ObTxLSStateMgr::is_follower() const +{ + LOAD_CUR_STATE_CONTAINER + return ls_state_container.state_val_.state_ == TxLSState::F_WORKING + || ls_state_container.state_val_.state_ == TxLSState::F_SWL_PENDING; +} + +bool ObTxLSStateMgr::is_block_start_tx() const +{ + LOAD_CUR_STATE_CONTAINER + return ls_state_container.state_val_.block_start_tx_flag_; +} + +bool ObTxLSStateMgr::is_block_start_readonly() const +{ + LOAD_CUR_STATE_CONTAINER + return ls_state_container.state_val_.block_start_readonly_flag_; +} + +bool ObTxLSStateMgr::is_block_start_normal_tx() const +{ + LOAD_CUR_STATE_CONTAINER + return ls_state_container.state_val_.block_start_normal_tx_flag_; +} + +bool ObTxLSStateMgr::is_block_WR() const +{ + LOAD_CUR_STATE_CONTAINER + return ls_state_container.state_val_.block_start_tx_flag_ + || ls_state_container.state_val_.block_start_readonly_flag_; +} + +bool ObTxLSStateMgr::is_leader_takeover_pending() const +{ + LOAD_CUR_STATE_CONTAINER + return is_switch_leader_pending_(ls_state_container) + || is_resume_leader_pending_(ls_state_container); +} + +bool ObTxLSStateMgr::is_start_working_apply_pending() const +{ + LOAD_CUR_STATE_CONTAINER + return ls_state_container.state_val_.state_ == TxLSState::T_APPLY_PENDING + || ls_state_container.state_val_.state_ == TxLSState::R_APPLY_PENDING; +} + +bool ObTxLSStateMgr::is_switch_leader_pending() const +{ + LOAD_CUR_STATE_CONTAINER + return is_switch_leader_pending_(ls_state_container); +} + +bool ObTxLSStateMgr::is_resume_leader_pending() const +{ + LOAD_CUR_STATE_CONTAINER + return is_resume_leader_pending_(ls_state_container); +} + +bool ObTxLSStateMgr::is_stopped() const +{ + LOAD_CUR_STATE_CONTAINER + return ls_state_container.state_val_.state_ == TxLSState::STOPPED; +} + +int ObTxLSStateMgr::execute_tx_ls_action_(const TxLSAction action) +{ + int ret = OB_SUCCESS; + + TxLSStateContainer res_state; + res_state.state_container_ = 0; + TxStateActionPair pair; + pair.base_state_ = cur_state_; + pair.exec_action_ = action; + + TxLSStateHashFunc state_func; + + ret = state_func(pair, res_state); + + if (OB_SUCCESS != ret || UINT64_MAX == res_state.state_container_) { + TRANS_LOG(WARN, "[Tx LS State] switch ls tx state failed", K(ret), K(pair), + K(res_state.state_val_), KPC(this)); + } else { + ATOMIC_STORE(&prev_state_.state_container_, pair.base_state_.state_container_); + ATOMIC_STORE(&cur_state_.state_container_, res_state.state_container_); + TRANS_LOG(INFO, "[Tx LS State] switch ls tx state successfully", K(ret), K(pair), + K(res_state.state_val_), KPC(this)); + } + + return ret; +} + +void ObTxLSStateMgr::clear_all_flag_(TxLSStateContainer &cur_state) +{ + cur_state.state_val_.offline_flag_ = false; + cur_state.state_val_.block_start_tx_flag_ = false; + cur_state.state_val_.block_start_readonly_flag_ = false; + cur_state.state_val_.block_start_normal_tx_flag_ = false; +} + +bool ObTxLSStateMgr::is_switch_leader_pending_(const TxLSStateContainer &ls_state_container) +{ + return ls_state_container.state_val_.state_ == TxLSState::T_SYNC_PENDING + || ls_state_container.state_val_.state_ == TxLSState::T_SYNC_FAILED + || ls_state_container.state_val_.state_ == TxLSState::T_APPLY_PENDING; +} + +bool ObTxLSStateMgr::is_resume_leader_pending_(const TxLSStateContainer &ls_state_container) +{ + + return ls_state_container.state_val_.state_ == TxLSState::R_SYNC_PENDING + || ls_state_container.state_val_.state_ == TxLSState::R_SYNC_FAILED + || ls_state_container.state_val_.state_ == TxLSState::R_APPLY_PENDING; +} + +} // namespace transaction + +} // namespace oceanbase diff --git a/src/storage/tx/ob_tx_ls_state_mgr.h b/src/storage/tx/ob_tx_ls_state_mgr.h new file mode 100644 index 0000000000..36904af1ad --- /dev/null +++ b/src/storage/tx/ob_tx_ls_state_mgr.h @@ -0,0 +1,339 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_TRANSACTION_OB_TX_LS_STATE_MGR +#define OCEANBASE_TRANSACTION_OB_TX_LS_STATE_MGR + +#include "lib/hash/ob_hashmap.h" +#include "ob_trans_define.h" + +/*** + * The Binary Representation of the LS State + * + * | ... Reserved Bits (44) | + * | Block Start Tx Flag (1) | Block Start Normal Tx Flag (1) | Block Start ReadOnly Flag (1) | + * | Offline Flag (1) | + * | TxLSState (16) | + */ + +/*** + * State Machine + * + * @Init: + * +--------------------+------------+----------+---------------+--------------+ + * | BaseState | BaseFlag | Action | TargetState | TargetFlag | + * +====================+============+==========+===============+==============+ + * | INIT | NONE | START | F_WORKING | NONE | + * +--------------------+------------+----------+---------------+--------------+ + * | INIT | NONE | STOP | INIT | NONE | + * +--------------------+------------+----------+---------------+--------------+ + * | STOPPED | UNLIMITED | STOP | STOPPED | NONE | + * +--------------------+------------+----------+---------------+--------------+ + * | F_WORKING | UNLIMITED | STOP | STOPPED | NONE | + * +--------------------+------------+----------+---------------+--------------+ + * | L_WORKING | UNLIMITED | STOP | STOPPED | NONE | + * +--------------------+------------+----------+---------------+--------------+ + * | T_SYNC_PENDING | UNLIMITED | STOP | STOPPED | NONE | + * +--------------------+------------+----------+---------------+--------------+ + * | R_SYNC_PENDING | UNLIMITED | STOP | STOPPED | NONE | + * +--------------------+------------+----------+---------------+--------------+ + * | T_APPLYING_PENDING | UNLIMITED | STOP | STOPPED | NONE | + * +--------------------+------------+----------+---------------+--------------+ + * | R_APPLYING_PENDING | UNLIMITED | STOP | STOPPED | NONE | + * +--------------------+------------+----------+---------------+--------------+ + * + * @Leader Takeover: + * +-----------------+------------+-----------------+-----------------+--------------+ + * | BaseState | BaseFlag | Action | TargetState | TargetFlag | + * +=================+============+=================+=================+==============+ + * | F_WORKING | UNLIMITED | LEADER_TAKEOVER | T_SYNC_PENDING | UNLIMITED | + * +-----------------+------------+-----------------+-----------------+--------------+ + * | F_WORKING | UNLIMITED | RESUME_LEADER | R_SYNC_PENDING | UNLIMITED | + * +-----------------+------------+-----------------+-----------------+--------------+ + * | T_SYNC_PENDING | UNLIMITED | SWL_CB_SUCC | T_APPLY_PENDING | UNLIMITED | + * +-----------------+------------+-----------------+-----------------+--------------+ + * | R_SYNC_PENDING | UNLIMITED | SWL_CB_SUCC | R_APPLY_PENDING | UNLIMITED | + * +-----------------+------------+-----------------+-----------------+--------------+ + * | T_SYNC_PENDING | UNLIMITED | SWL_CB_FAIL | T_SYNC_FAILED | UNLIMITED | + * +-----------------+------------+-----------------+-----------------+--------------+ + * | R_SYNC_PENDING | UNLIMITED | SWL_CB_FAIL | R_SYNC_FAILED | UNLIMITED | + * +-----------------+------------+-----------------+-----------------+--------------+ + * | T_APPLY_PENDING | UNLIMITED | APPLY_SWL_SUCC | L_WORKING | UNLIMITED | + * +-----------------+------------+-----------------+-----------------+--------------+ + * | R_APPLY_PENDING | UNLIMITED | APPLY_SWL_SUCC | L_WORKING | UNLIMITED | + * +-----------------+------------+-----------------+-----------------+--------------+ + * | T_APPLY_PENDING | UNLIMITED | APPLY_SWL_FAIL | T_APPLY_PENDING | UNLIMITED | + * +-----------------+------------+-----------------+-----------------+--------------+ + * | R_APPLY_PENDING | UNLIMITED | APPLY_SWL_FAIL | R_APPLY_PENDING | UNLIMITED | + * +-----------------+------------+-----------------+-----------------+--------------+ + * + * @Leader Revoke: + * +-----------------+------------+--------------------------+---------------+-------------------------------+ + * | BaseState | BaseFlag | Action | TargetState | TargetFlag | + * +=================+============+==========================+===============+===============================+ + * | L_WORKING | UNLIMITED | LEADER_REVOKE_GRACEFULLY | F_WORKING | BLOCK_STATR_NORMAL_TX = False | + * +-----------------+------------+--------------------------+---------------+-------------------------------+ + * | L_WORKING | UNLIMITED | LEADER_REVOKE_FORCEDLLY | F_WORKING | BLOCK_STATR_NORMAL_TX = False | + * +-----------------+------------+--------------------------+---------------+-------------------------------+ + * | T_SYNC_FAILED | UNLIMITED | LEADER_REVOKE_GRACEFULLY | F_WORKING | BLOCK_STATR_NORMAL_TX = False | + * +-----------------+------------+--------------------------+---------------+-------------------------------+ + * | R_SYNC_FAILED | UNLIMITED | LEADER_REVOKE_GRACEFULLY | F_WORKING | BLOCK_STATR_NORMAL_TX = False | + * +-----------------+------------+--------------------------+---------------+-------------------------------+ + * | T_SYNC_FAILED | UNLIMITED | LEADER_REVOKE_FORCEDLLY | F_WORKING | BLOCK_STATR_NORMAL_TX = False | + * +-----------------+------------+--------------------------+---------------+-------------------------------+ + * | R_SYNC_FAILED | UNLIMITED | LEADER_REVOKE_FORCEDLLY | F_WORKING | BLOCK_STATR_NORMAL_TX = False | + * +-----------------+------------+--------------------------+---------------+-------------------------------+ + * | T_APPLY_PENDING | UNLIMITED | LEADER_REVOKE_FORCEDLLY | F_SWL_PENDING | BLOCK_STATR_NORMAL_TX = False | + * +-----------------+------------+--------------------------+---------------+-------------------------------+ + * | R_APPLY_PENDING | UNLIMITED | LEADER_REVOKE_FORCEDLLY | F_SWL_PENDING | BLOCK_STATR_NORMAL_TX = False | + * +-----------------+------------+--------------------------+---------------+-------------------------------+ + * | F_SWL_PENDING | UNLIMITED | APPLY_SWL_FAIL | F_SWL_PENDING | BLOCK_STATR_NORMAL_TX = False | + * +-----------------+------------+--------------------------+---------------+-------------------------------+ + * | F_SWL_PENDING | UNLIMITED | APPLY_SWL_SUCC | F_WORKING | BLOCK_STATR_NORMAL_TX = False | + * +-----------------+------------+--------------------------+---------------+-------------------------------+ + * + * @Offline: + * +-------------+------------+----------+---------------+-----------------------------------------------------------------------+ + * | BaseState | BaseFlag | Action | TargetState | TargetFlag | + * +=============+============+==========+===============+=======================================================================+ + * | UNLIMITED | UNLIMITED | OFFLINE | UNLIMITED | OFFLINE = True | + * +-------------+------------+----------+---------------+-----------------------------------------------------------------------+ + * | UNLIMITED | UNLIMITED | ONLINE | UNLIMITED | OFFLINE = False, BLOCK_START_TX = False, BLOCK_START_READONLY = False | + * +-------------+------------+----------+---------------+-----------------------------------------------------------------------+ + * Notion: Before 4.3, the ls may be started to working without online operation. + * + * @Block Tx + * +-------------+------------+-----------------------+---------------+-------------------------------+ + * | BaseState | BaseFlag | Action | TargetState | TargetFlag | + * +=============+============+=======================+===============+===============================+ + * | UNLIMITED | UNLIMITED | BLOCK_START_TX | UNLIMITED | BLOCK_START_TX = True | + * +-------------+------------+-----------------------+---------------+-------------------------------+ + * | UNLIMITED | UNLIMITED | BLOCK_START_READONLY | UNLIMITED | BLOCK_START_READONLY = True | + * +-------------+------------+-----------------------+---------------+-------------------------------+ + * | L_WORKING | UNLIMITED | BLOCK_START_NORMAL_TX | UNLIMITED | BLOCK_START_NORMAL_TX = True | + * +-------------+------------+-----------------------+---------------+-------------------------------+ + * | L_WORKING | UNLIMITED | UNBLOCK_NORMAL_TX | UNLIMITED | BLOCK_START_NORMAL_TX = False | + * +-------------+------------+-----------------------+---------------+-------------------------------+ + */ + +namespace oceanbase +{ + +namespace transaction +{ + +class ObTxLSStateMgr +{ + +public: + enum TxLSAction : uint64_t + { + UNKOWN = 0, + START = 1, + STOP = 2, + ONLINE = 3, + OFFLINE = 4, + LEADER_REVOKE_GRACEFULLY = 5, + LEADER_REVOKE_FORCEDLLY = 6, + SWL_CB_SUCC = 7, + SWL_CB_FAIL = 8, + APPLY_SWL_SUCC = 9, + APPLY_SWL_FAIL = 10, + LEADER_TAKEOVER = 12, + RESUME_LEADER = 13, + BLOCK_START_TX = 14, + BLOCK_START_READONLY = 15, + BLOCK_START_NORMAL_TX = 16, + UNBLOCK_NORMAL_TX = 17, + BLOCK_START_WR = 18 + }; + + enum TxLSState : uint16_t + { + INIT = 0, + F_WORKING = 1, + L_WORKING = 4, + STOPPED = 14, + T_SYNC_PENDING = 15, + R_SYNC_PENDING = 16, + T_SYNC_FAILED = 17, + R_SYNC_FAILED = 18, + T_APPLY_PENDING = 19, + R_APPLY_PENDING = 20, + F_SWL_PENDING = 21 + }; + +#define TCM_STATE_CASE_TO_STR(state) \ + case state: \ + str = #state; \ + break; + + static const char *state_str(uint16_t state) + { + const char *str = "INVALID"; + switch (state) { + TCM_STATE_CASE_TO_STR(INIT); + TCM_STATE_CASE_TO_STR(F_WORKING); + TCM_STATE_CASE_TO_STR(L_WORKING); + TCM_STATE_CASE_TO_STR(STOPPED); + TCM_STATE_CASE_TO_STR(T_SYNC_PENDING); + TCM_STATE_CASE_TO_STR(R_SYNC_PENDING); + TCM_STATE_CASE_TO_STR(T_SYNC_FAILED); + TCM_STATE_CASE_TO_STR(R_SYNC_FAILED); + TCM_STATE_CASE_TO_STR(T_APPLY_PENDING); + TCM_STATE_CASE_TO_STR(R_APPLY_PENDING); + TCM_STATE_CASE_TO_STR(F_SWL_PENDING); + + default: + break; + } + return str; + } + + static const char *action_str(TxLSAction action) + { + const char *str = "INVALID"; + switch (action) { + TCM_STATE_CASE_TO_STR(UNKOWN); + TCM_STATE_CASE_TO_STR(START); + TCM_STATE_CASE_TO_STR(STOP); + TCM_STATE_CASE_TO_STR(ONLINE); + TCM_STATE_CASE_TO_STR(OFFLINE); + TCM_STATE_CASE_TO_STR(LEADER_REVOKE_GRACEFULLY); + TCM_STATE_CASE_TO_STR(LEADER_REVOKE_FORCEDLLY); + TCM_STATE_CASE_TO_STR(SWL_CB_SUCC); + TCM_STATE_CASE_TO_STR(SWL_CB_FAIL); + TCM_STATE_CASE_TO_STR(APPLY_SWL_SUCC); + TCM_STATE_CASE_TO_STR(APPLY_SWL_FAIL); + TCM_STATE_CASE_TO_STR(LEADER_TAKEOVER); + TCM_STATE_CASE_TO_STR(RESUME_LEADER); + TCM_STATE_CASE_TO_STR(BLOCK_START_TX); + TCM_STATE_CASE_TO_STR(BLOCK_START_READONLY); + TCM_STATE_CASE_TO_STR(BLOCK_START_NORMAL_TX); + TCM_STATE_CASE_TO_STR(UNBLOCK_NORMAL_TX); + TCM_STATE_CASE_TO_STR(BLOCK_START_WR); + default: + break; + } + return str; + } +#undef TCM_STATE_CASE_TO_STR + + union TxLSStateContainer + { + struct StateVal + { + uint16_t state_ : 16; + uint8_t offline_flag_ : 1; + uint8_t block_start_tx_flag_ : 1; + uint8_t block_start_normal_tx_flag_ : 1; + uint8_t block_start_readonly_flag_ : 1; + uint64_t reserved_bit_ : 44; + + TO_STRING_KV("state_", + state_str(state_), + K(offline_flag_), + K(block_start_tx_flag_), + K(block_start_normal_tx_flag_), + K(block_start_readonly_flag_)); + } state_val_; + uint64_t state_container_; + }; + + struct TxStateActionPair + { + TxLSStateContainer base_state_; + TxLSAction exec_action_; + + TxStateActionPair() { reset(); } + void reset() + { + base_state_.state_container_ = 0; + exec_action_ = TxLSAction::UNKOWN; + } + + TO_STRING_KV(K(base_state_.state_val_), "exec_action_", action_str(exec_action_)); + }; + struct TxLSStateHashFunc + { + int operator()(const TxStateActionPair &key, TxLSStateContainer &res) const; + }; + + // typedef common::hash:: + // ObHashMap + // StateActionHashMap; + +public: + ObTxLSStateMgr() { reset(); } + + int init(const share::ObLSID ls_id) + { + ls_id_ = ls_id; + return OB_SUCCESS; + } + + void reset() + { + ls_id_.reset(); + cur_state_.state_container_ = 0; + prev_state_.state_container_ = 0; + } + int switch_tx_ls_state(const TxLSAction action, + const share::SCN start_working_scn = share::SCN::invalid_scn()); + void restore_tx_ls_state(); + + bool need_retry_apply_SWL(share::SCN &applying_swl_scn); + bool waiting_SWL_cb(); + void replay_SWL_succ(const share::SCN & swl_scn); + + const char * + iter_ctx_mgr_stat_info(uint64_t &state_container, bool &is_master, bool &is_stopped) const; + // TxLSState get_state() const; + bool is_master() const; + bool is_follower() const; + bool is_block_start_readonly() const; + bool is_block_start_tx() const; + bool is_block_start_normal_tx() const; + bool is_block_WR() const; + bool is_start_working_apply_pending() const; + bool is_leader_takeover_pending() const; + bool is_switch_leader_pending() const; + bool is_resume_leader_pending() const; + bool is_stopped() const; + + TO_STRING_KV(K(ls_id_), + K(cur_state_.state_val_), + K(prev_state_.state_val_), + K(max_applying_start_working_ts_), + K(max_applied_start_working_ts_)); + +private: + int execute_tx_ls_action_(const TxLSAction action); + static void clear_all_flag_(TxLSStateContainer &cur_state); + + static bool is_switch_leader_pending_(const TxLSStateContainer &ls_state_container); + static bool is_resume_leader_pending_(const TxLSStateContainer &ls_state_container); + +private: + share::ObLSID ls_id_; + TxLSStateContainer cur_state_; + TxLSStateContainer prev_state_; + + share::SCN max_applied_start_working_ts_; + share::SCN max_applying_start_working_ts_; +}; + +} // namespace transaction + +} // namespace oceanbase +#endif diff --git a/src/storage/tx/ob_tx_replay_executor.cpp b/src/storage/tx/ob_tx_replay_executor.cpp index 8788add388..5d94c320e9 100644 --- a/src/storage/tx/ob_tx_replay_executor.cpp +++ b/src/storage/tx/ob_tx_replay_executor.cpp @@ -314,7 +314,7 @@ int ObTxReplayExecutor::try_get_tx_ctx_() } else if (base_header_.need_pre_replay_barrier() && OB_UNLIKELY(ctx_->is_replay_complete_unknown())) { // if a pre-barrier log will be replayed // the txn can be confirmed to incomplete replayed - ret = ctx_->set_replay_incomplete(); + ret = ctx_->set_replay_incomplete(log_ts_ns_); } } } diff --git a/src/storage/tx_table/ob_tx_data_cache.cpp b/src/storage/tx_table/ob_tx_data_cache.cpp index eb2454da16..fade20b3bb 100644 --- a/src/storage/tx_table/ob_tx_data_cache.cpp +++ b/src/storage/tx_table/ob_tx_data_cache.cpp @@ -12,6 +12,7 @@ #include "ob_tx_data_cache.h" #include "share/rc/ob_tenant_base.h" +#include "storage/tx/ob_tx_data_op.h" namespace oceanbase { namespace storage { @@ -30,7 +31,7 @@ int ObTxDataCacheValue::init(const ObTxData &tx_data) STORAGE_LOG(WARN, "init tx data cache value twice", KR(ret), KPC(this)); } else { // reserve or allocate buf to store tx data - int64_t size = tx_data.size(); + int64_t size = tx_data.size_need_cache(); void *tx_data_buf = nullptr; if (TX_DATA_SLICE_SIZE == size) { // this tx data do not have undo actions, use reserved memory @@ -114,17 +115,24 @@ int ObTxDataCacheValue::inner_deep_copy_(void *tx_data_buf, const ObTxData &rhs) } else { tx_data_ = new (tx_data_buf) ObTxData(); tx_data_->assign_without_undo(rhs); + tx_data_->tx_data_allocator_ = rhs.tx_data_allocator_; + tx_data_->op_allocator_ = rhs.op_allocator_; + if (rhs.op_guard_.is_valid()) { + ObTxDataOp *tx_data_op = new ((char*)tx_data_buf + TX_DATA_SLICE_SIZE) ObTxDataOp(tx_data_->tx_data_allocator_, + tx_data_->op_allocator_); + tx_data_->op_guard_.init(tx_data_op); + } - if (OB_LIKELY(nullptr == rhs.undo_status_list_.head_)) { + if (OB_LIKELY(!rhs.op_guard_.is_valid() || OB_ISNULL(rhs.op_guard_->get_undo_status_list().head_))) { // this tx data do not have undo status } else { - undo_node_array_ = (ObUndoStatusNode *)((char *)tx_data_buf + TX_DATA_SLICE_SIZE); + undo_node_array_ = (ObUndoStatusNode *)((char *)tx_data_buf + TX_DATA_SLICE_SIZE + TX_DATA_SLICE_SIZE); + // ignore mds op if (OB_FAIL(inner_deep_copy_undo_status_(rhs))) { STORAGE_LOG(WARN, "deep copy undo status node for tx data kv cache failed", KR(ret), K(rhs)); } } } - return ret; } @@ -134,21 +142,21 @@ int ObTxDataCacheValue::inner_deep_copy_undo_status_(const ObTxData &rhs) // use dummy head point to the first undo node ObUndoStatusNode dummy_head; - dummy_head.next_ = rhs.undo_status_list_.head_; + dummy_head.next_ = rhs.op_guard_->get_undo_status_list().head_; ObUndoStatusNode *pre_node = &dummy_head; - tx_data_->undo_status_list_.undo_node_cnt_ = rhs.undo_status_list_.undo_node_cnt_; - for (int64_t i = 0; OB_SUCC(ret) && i < rhs.undo_status_list_.undo_node_cnt_; i++) { + tx_data_->op_guard_->get_undo_status_list().undo_node_cnt_ = rhs.op_guard_->get_undo_status_list().undo_node_cnt_; + for (int64_t i = 0; OB_SUCC(ret) && i < rhs.op_guard_->get_undo_status_list().undo_node_cnt_; i++) { ObUndoStatusNode *rhs_node = pre_node->next_; pre_node = rhs_node; if (OB_ISNULL(rhs_node)) { ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "undo status list node count dismatach", KR(ret), K(i), K(rhs.undo_status_list_.undo_node_cnt_)); + STORAGE_LOG(ERROR, "undo status list node count dismatach", KR(ret), K(i), K(rhs.op_guard_->get_undo_status_list().undo_node_cnt_)); } else { undo_node_array_[i].assign_value(*rhs_node); undo_node_array_[i].next_ = nullptr; if (0 == i) { - tx_data_->undo_status_list_.head_ = undo_node_array_; + tx_data_->op_guard_->get_undo_status_list().head_ = undo_node_array_; } else { undo_node_array_[i-1].next_ = &undo_node_array_[i]; } @@ -192,4 +200,4 @@ int ObTxDataKVCache::put_row(const ObTxDataCacheKey &key, const ObTxDataCacheVal } // namespace storage -} // namespace oceanbase \ No newline at end of file +} // namespace oceanbase diff --git a/src/storage/tx_table/ob_tx_data_cache.h b/src/storage/tx_table/ob_tx_data_cache.h index 6a96cf5ed6..fc6c7917de 100644 --- a/src/storage/tx_table/ob_tx_data_cache.h +++ b/src/storage/tx_table/ob_tx_data_cache.h @@ -107,7 +107,7 @@ public: TO_STRING_KV(K_(is_inited), KP_(tx_data), KPC_(tx_data), KP_(mtl_alloc_buf), KP(&reserved_buf_)); public: // derived from ObIKVCacheValue - virtual int64_t size() const { return (IS_INIT && OB_NOT_NULL(tx_data_)) ? sizeof(*this) + tx_data_->size() : 0; } + virtual int64_t size() const { return (IS_INIT && OB_NOT_NULL(tx_data_)) ? sizeof(*this) + tx_data_->size_need_cache() : 0; } virtual int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheValue *&value) const; diff --git a/src/storage/tx_table/ob_tx_data_hash_map.cpp b/src/storage/tx_table/ob_tx_data_hash_map.cpp index 1d3b76fdb2..bc10d79bd5 100644 --- a/src/storage/tx_table/ob_tx_data_hash_map.cpp +++ b/src/storage/tx_table/ob_tx_data_hash_map.cpp @@ -57,7 +57,6 @@ int ObTxDataHashMap::init() int ObTxDataHashMap::insert(const transaction::ObTransID &key, ObTxData *value) { int ret = OB_SUCCESS; - if (!key.is_valid() || OB_ISNULL(value)) { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "invalid argument", K(key), KP(value)); @@ -161,4 +160,4 @@ int ObTxDataHashMap::Iterator::get_next(ObTxDataGuard &guard) } } // namespace storage -} // namespace oceanbase \ No newline at end of file +} // namespace oceanbase diff --git a/src/storage/tx_table/ob_tx_data_memtable.cpp b/src/storage/tx_table/ob_tx_data_memtable.cpp index 5c46ccfa17..5d0b659a7f 100644 --- a/src/storage/tx_table/ob_tx_data_memtable.cpp +++ b/src/storage/tx_table/ob_tx_data_memtable.cpp @@ -182,15 +182,15 @@ int ObTxDataMemtable::insert(ObTxData *tx_data) max_tx_scn_.inc_update(tx_data->end_scn_); atomic_update_(tx_data); ATOMIC_INC(&inserted_cnt_); - if (OB_UNLIKELY(tx_data->undo_status_list_.undo_node_cnt_ >= 10)) { - if (tx_data->undo_status_list_.undo_node_cnt_ == 10 || tx_data->undo_status_list_.undo_node_cnt_ % 100 == 0) { + if (OB_UNLIKELY(tx_data->op_guard_.is_valid() && tx_data->op_guard_->get_undo_status_list().undo_node_cnt_ >= 10)) { + if (tx_data->op_guard_->get_undo_status_list().undo_node_cnt_ == 10 || tx_data->op_guard_->get_undo_status_list().undo_node_cnt_ % 100 == 0) { STORAGE_LOG(INFO, "attention! this tx write too many rollback to savepoint log", "ls_id", get_ls_id(), "tx_id", tx_data->tx_id_, "state", ObTxData::get_state_string(tx_data->state_), - "undo_node_cnt", tx_data->undo_status_list_.undo_node_cnt_, - "newest_undo_node", tx_data->undo_status_list_.head_, + "undo_node_cnt", tx_data->op_guard_->get_undo_status_list().undo_node_cnt_, + "newest_undo_node", tx_data->op_guard_->get_undo_status_list().head_, K(tx_data->start_scn_), K(tx_data->end_scn_)); } @@ -205,9 +205,21 @@ void ObTxDataMemtable::atomic_update_(ObTxData *tx_data) int64_t thread_idx = common::get_itid() & MAX_CONCURRENCY_MOD_MASK; min_tx_scn_[thread_idx].dec_update(tx_data->end_scn_); min_start_scn_[thread_idx].dec_update(tx_data->start_scn_); - int64_t tx_data_size = TX_DATA_SLICE_SIZE * (1LL + tx_data->undo_status_list_.undo_node_cnt_); + int64_t tx_data_size = 0; + int64_t count = 0; + if (tx_data->state_ == ObTxCommitData::RUNNING) { + tx_data_size = TX_DATA_SLICE_SIZE; + } else if (!tx_data->op_guard_.is_valid()) { + tx_data_size = TX_DATA_SLICE_SIZE; + } else { + count = tx_data->op_guard_->get_undo_status_list().undo_node_cnt_; + int64_t tx_op_size = tx_data->op_guard_->get_tx_op_size(); + tx_data_size = TX_DATA_SLICE_SIZE + TX_DATA_SLICE_SIZE + + count * TX_DATA_SLICE_SIZE + // undo status list + tx_op_size; // tx_op + } ATOMIC_FAA(&occupied_size_[thread_idx], tx_data_size); - ATOMIC_FAA(&total_undo_node_cnt_[thread_idx], tx_data->undo_status_list_.undo_node_cnt_); + ATOMIC_FAA(&total_undo_node_cnt_[thread_idx], count); } int ObTxDataMemtable::get_tx_data(const ObTransID &tx_id, ObTxDataGuard &tx_data_guard) @@ -377,7 +389,6 @@ int ObTxDataMemtable::pre_process_commit_version_row_(ObTxData *fake_tx_data) fake_tx_data->tx_id_ = INT64_MAX; fake_tx_data->commit_version_.convert_for_tx(serialize_size); fake_tx_data->start_scn_.convert_for_tx((int64_t)buf_.get_ptr()); - fake_tx_data->undo_status_list_.head_ = nullptr; } } @@ -648,13 +659,11 @@ int ObTxDataMemtable::estimate_phy_size(const ObStoreRowkey *start_key, return ret; } -int ObTxDataMemtable::get_split_ranges(const ObStoreRowkey *start_key, - const ObStoreRowkey *end_key, +int ObTxDataMemtable::get_split_ranges(const ObStoreRange &input_range, const int64_t part_cnt, common::ObIArray &range_array) { - UNUSED(start_key); - UNUSED(end_key); + UNUSED(input_range); int ret = OB_SUCCESS; if (!pre_process_done_) { diff --git a/src/storage/tx_table/ob_tx_data_memtable.h b/src/storage/tx_table/ob_tx_data_memtable.h index 9a74d63fa8..f875ec13d3 100644 --- a/src/storage/tx_table/ob_tx_data_memtable.h +++ b/src/storage/tx_table/ob_tx_data_memtable.h @@ -19,6 +19,7 @@ #include "storage/tx/ob_tx_data_define.h" #include "storage/tx_table/ob_tx_table_define.h" #include "storage/tx_table/tx_table_local_buffer.h" +#include "storage/tx/ob_tx_data_op.h" namespace oceanbase { @@ -296,8 +297,7 @@ public: /* derived from ObIMemtable */ int64_t &total_bytes, int64_t &total_rows) override; - virtual int get_split_ranges(const ObStoreRowkey *start_key, - const ObStoreRowkey *end_key, + virtual int get_split_ranges(const ObStoreRange &input_range, const int64_t part_cnt, common::ObIArray &range_array) override; // not supported @@ -314,7 +314,6 @@ public: // checkpoint } int flush(const int64_t trace_id); - /** * @brief Because of the random order of clog callbacks, the tx data in a freezing tx data * memtable may not completed. We must wait until the max_consequent_callbacked_scn is larger @@ -512,7 +511,10 @@ public: // printf undo status list fprintf(fd_, "Undo Actions [from, to): {"); - ObUndoStatusNode *cur_node = tx_data->undo_status_list_.head_; + ObUndoStatusNode *cur_node = NULL; + if (tx_data->op_guard_.is_valid()) { + cur_node = tx_data->op_guard_->get_undo_status_list().head_; + } while (OB_NOT_NULL(cur_node)) { for (int i = 0; i < cur_node->size_; i++) { diff --git a/src/storage/tx_table/ob_tx_data_table.cpp b/src/storage/tx_table/ob_tx_data_table.cpp index 8c93f888f0..5b484a4bbe 100644 --- a/src/storage/tx_table/ob_tx_data_table.cpp +++ b/src/storage/tx_table/ob_tx_data_table.cpp @@ -264,72 +264,12 @@ int ObTxDataTable::alloc_tx_data(ObTxDataGuard &tx_data_guard, } else { ObTxData *tx_data = new (slice_ptr) ObTxData(); tx_data->tx_data_allocator_ = tx_data_allocator_; + tx_data->op_allocator_ = &MTL(share::ObSharedMemAllocMgr*)->tx_data_op_allocator(); tx_data_guard.init(tx_data); } return ret; } -int ObTxDataTable::deep_copy_tx_data(const ObTxDataGuard &in_tx_data_guard, ObTxDataGuard &out_tx_data_guard) -{ - int ret = OB_SUCCESS; - void *slice_ptr = nullptr; - const int64_t abs_expire_time = THIS_WORKER.get_timeout_ts(); - const ObTxData *in_tx_data = in_tx_data_guard.tx_data(); - ObTxData *out_tx_data = nullptr; - - if (OB_ISNULL(slice_ptr = tx_data_allocator_->alloc(true, abs_expire_time))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "allocate memory from slice_allocator fail.", KR(ret), KP(this), - K(tablet_id_), K(abs_expire_time)); - } else if (OB_ISNULL(in_tx_data)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "invalid nullptr of tx data", K(in_tx_data_guard), KPC(this)); - } else { - out_tx_data = new (slice_ptr) ObTxData(); - *out_tx_data = *in_tx_data; - out_tx_data->tx_data_allocator_ = tx_data_allocator_; - out_tx_data->undo_status_list_.head_ = nullptr; - out_tx_data->ref_cnt_ = 0; - out_tx_data_guard.init(out_tx_data); - - if (OB_FAIL(deep_copy_undo_status_list_(in_tx_data->undo_status_list_, - out_tx_data->undo_status_list_))) { - STORAGE_LOG(WARN, "deep copy undo status list failed."); - } else { - // deep copy succeed. - } - } - return ret; -} - -int ObTxDataTable::deep_copy_undo_status_list_(const ObUndoStatusList &in_list, - ObUndoStatusList &out_list) -{ - int ret = OB_SUCCESS; - ObUndoStatusNode *cur_in_node = in_list.head_; - ObUndoStatusNode *pre_node = nullptr; - ObUndoStatusNode *new_node = nullptr; - - while (OB_SUCC(ret) && nullptr != cur_in_node) { - if (OB_FAIL(alloc_undo_status_node(new_node))) { - STORAGE_LOG(WARN, "alloc undo status node failed.", KR(ret)); - } else { - *new_node = *cur_in_node; - // reset next pointer to avoid invalid free - new_node->next_ = nullptr; - if (nullptr == pre_node) { - out_list.head_ = new_node; - } else { - pre_node->next_ = new_node; - } - pre_node = new_node; - cur_in_node = cur_in_node->next_; - } - } - - return ret; -} - int ObTxDataTable::alloc_undo_status_node(ObUndoStatusNode *&undo_status_node) { int ret = OB_SUCCESS; @@ -974,11 +914,11 @@ int ObTxDataTable::DEBUG_calc_with_row_iter_(ObStoreRowIterator *row_iter, tmp_upper_trans_version = tx_data.commit_version_; } } - - if (OB_NOT_NULL(tx_data.undo_status_list_.head_)) { - free_undo_status_list_(tx_data.undo_status_list_.head_); - tx_data.undo_status_list_.head_ = nullptr; + if (tx_data.op_guard_.is_valid() && OB_NOT_NULL(tx_data.op_guard_->get_undo_status_list().head_)) { + free_undo_status_list_(tx_data.op_guard_->get_undo_status_list().head_); + tx_data.op_guard_->get_undo_status_list().head_ = nullptr; } + } if (OB_ITER_END == ret) { @@ -1137,6 +1077,14 @@ int ObTxDataTable::check_min_start_in_tx_data_(const SCN &sstable_end_scn, return ret; } +int ObTxDataTable::deep_copy_tx_data(const ObTxDataGuard &in_tx_data, ObTxDataGuard &out_tx_data) +{ + int ret = OB_NOT_SUPPORTED; + UNUSED(in_tx_data); + UNUSED(out_tx_data); + return ret; +} + int ObTxDataTable::update_cache_if_needed_(bool &skip_calc) { int ret = OB_SUCCESS; @@ -1243,7 +1191,7 @@ int ObTxDataTable::calc_upper_trans_scn_(const SCN sstable_end_scn, SCN &upper_t return ret; } -int ObTxDataTable::supplement_undo_actions_if_exist(ObTxData *tx_data) +int ObTxDataTable::supplement_tx_op_if_exist(ObTxData *tx_data) { int ret = OB_SUCCESS; ObTxData tx_data_from_sstable; @@ -1256,6 +1204,8 @@ int ObTxDataTable::supplement_undo_actions_if_exist(ObTxData *tx_data) } else if (OB_ISNULL(tx_data)) { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(ERROR, "tx data is nullptr", KR(ret), KP(this)); + } else if (FALSE_IT(tx_data_from_sstable.tx_data_allocator_ = &MTL(share::ObSharedMemAllocMgr*)->tx_data_allocator())) { + } else if (FALSE_IT(tx_data_from_sstable.op_allocator_ = &MTL(share::ObSharedMemAllocMgr*)->tx_data_op_allocator())) { } else if (OB_FAIL(get_tx_data_in_sstable_(tx_data->tx_id_, tx_data_from_sstable, unused_scn))) { if (ret == OB_TRANS_CTX_NOT_EXIST) { // This transaction does not have undo actions @@ -1263,19 +1213,10 @@ int ObTxDataTable::supplement_undo_actions_if_exist(ObTxData *tx_data) } else { STORAGE_LOG(WARN, "get tx data from sstable failed.", KR(ret)); } - } else { - // assign and reset to avoid deep copy - if (OB_NOT_NULL(tx_data->undo_status_list_.head_)) { - STORAGE_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "invalid undo status list", KPC(tx_data)); - } - tx_data->undo_status_list_ = tx_data_from_sstable.undo_status_list_; - tx_data_from_sstable.undo_status_list_.reset(); - } - - if (OB_NOT_NULL(tx_data_from_sstable.undo_status_list_.head_)) { - STORAGE_LOG(WARN, "supplement undo actions failed", KR(ret), KPC(tx_data), K(get_ls_id())); - free_undo_status_list_(tx_data_from_sstable.undo_status_list_.head_); + } else if (FALSE_IT(*tx_data = tx_data_from_sstable)) { } + tx_data_from_sstable.tx_data_allocator_ = nullptr; + tx_data_from_sstable.reset(); return ret; } diff --git a/src/storage/tx_table/ob_tx_data_table.h b/src/storage/tx_table/ob_tx_data_table.h index e5c75911c7..985dbecade 100644 --- a/src/storage/tx_table/ob_tx_data_table.h +++ b/src/storage/tx_table/ob_tx_data_table.h @@ -209,9 +209,9 @@ public: // ObTxDataTable int get_upper_trans_version_before_given_scn(const share::SCN sstable_end_scn, share::SCN &upper_trans_version); /** - * @brief see ObTxTable::supplement_undo_actions_if_exist + * @brief see ObTxTable::supplement_tx_op_if_exist */ - int supplement_undo_actions_if_exist(ObTxData *tx_data); + int supplement_tx_op_if_exist(ObTxData *tx_data); int self_freeze_task(); diff --git a/src/storage/tx_table/ob_tx_table.cpp b/src/storage/tx_table/ob_tx_table.cpp index f99446ee59..a12ecad574 100644 --- a/src/storage/tx_table/ob_tx_table.cpp +++ b/src/storage/tx_table/ob_tx_table.cpp @@ -645,7 +645,8 @@ int ObTxTable::check_with_tx_data(ObReadTxDataArg &read_tx_data_arg, ObITxDataCh // step 1 : read tx data in mini cache int tmp_ret = OB_SUCCESS; bool find_tx_data_in_cache = false; - if (OB_TMP_FAIL(check_tx_data_in_mini_cache_(read_tx_data_arg, fn))) { + if (read_tx_data_arg.skip_cache_) { + } else if (OB_TMP_FAIL(check_tx_data_in_mini_cache_(read_tx_data_arg, fn))) { if (OB_TRANS_CTX_NOT_EXIST != tmp_ret) { STORAGE_LOG(WARN, "check tx data in mini cache failed", KR(tmp_ret), K(read_tx_data_arg)); } @@ -655,7 +656,8 @@ int ObTxTable::check_with_tx_data(ObReadTxDataArg &read_tx_data_arg, ObITxDataCh } // step 2 : read tx data in kv cache - if (find_tx_data_in_cache) { + if (read_tx_data_arg.skip_cache_) { + } else if (find_tx_data_in_cache) { // already find tx data and do function with mini cache } else if (OB_TMP_FAIL(check_tx_data_in_kv_cache_(read_tx_data_arg, fn))) { if (OB_TRANS_CTX_NOT_EXIST != tmp_ret) { @@ -729,7 +731,7 @@ int ObTxTable::check_tx_data_in_kv_cache_(ObReadTxDataArg &read_tx_data_arg, ObI if (ObTxData::RUNNING == tx_data->state_) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(ERROR, "read an unexpected state tx data from kv cache"); - } else if (OB_ISNULL(tx_data->undo_status_list_.head_)) { + } else if (!tx_data->op_guard_.is_valid()) { // put into mini cache only if this tx data do not have undo actions read_tx_data_arg.tx_data_mini_cache_.set(*tx_data); } @@ -768,7 +770,7 @@ int ObTxTable::check_tx_data_in_tables_(ObReadTxDataArg &read_tx_data_arg, ObITx // if tx data is not null, put tx data into cache if (ObTxData::RUNNING == tx_data->state_) { } else { - if (OB_ISNULL(tx_data->undo_status_list_.head_)) { + if (!tx_data->op_guard_.is_valid()) { read_tx_data_arg.tx_data_mini_cache_.set(*tx_data); } @@ -1004,9 +1006,9 @@ int ObTxTable::cleanout_tx_node(ObReadTxDataArg &read_tx_data_arg, return ret; } -int ObTxTable::supplement_undo_actions_if_exist(ObTxData *tx_data) +int ObTxTable::supplement_tx_op_if_exist(ObTxData *tx_data) { - return tx_data_table_.supplement_undo_actions_if_exist(tx_data); + return tx_data_table_.supplement_tx_op_if_exist(tx_data); } int ObTxTable::self_freeze_task() { return tx_data_table_.self_freeze_task(); } @@ -1015,7 +1017,7 @@ int ObTxTable::generate_virtual_tx_data_row(const transaction::ObTransID tx_id, { GenerateVirtualTxDataRowFunctor fn(row_data); ObTxDataMiniCache mini_cache; - ObReadTxDataArg read_tx_data_arg(tx_id, epoch_, mini_cache); + ObReadTxDataArg read_tx_data_arg(tx_id, epoch_, mini_cache, true); int ret = check_with_tx_data(read_tx_data_arg, fn); return ret; } diff --git a/src/storage/tx_table/ob_tx_table.h b/src/storage/tx_table/ob_tx_table.h index d004f41eca..12ec5b154d 100644 --- a/src/storage/tx_table/ob_tx_table.h +++ b/src/storage/tx_table/ob_tx_table.h @@ -138,11 +138,11 @@ public: /** * @brief check whether the row key is locked by tx id - * - * @param[in] read_trans_id - * @param[in] data_trans_id - * @param[in] sql_sequence - * @param[out] lock_state + * + * @param[in] read_trans_id + * @param[in] data_trans_id + * @param[in] sql_sequence + * @param[out] lock_state */ int check_row_locked(ObReadTxDataArg &read_tx_data_arg, const transaction::ObTransID &read_tx_id, @@ -151,10 +151,10 @@ public: /** * @brief check whether transaction data_tx_id with sql_sequence is readable. (sql_sequence may be unreadable for txn or stmt rollback) - * - * @param[in] data_tx_id - * @param[in] sql_sequence - * @param[out] can_read + * + * @param[in] data_tx_id + * @param[in] sql_sequence + * @param[out] can_read */ int check_sql_sequence_can_read(ObReadTxDataArg &read_tx_data_arg, const transaction::ObTxSEQ &sql_sequence, @@ -243,7 +243,7 @@ public: * * @param[in & out] tx_data The pointer of tx data to be supplemented which is in tx ctx. */ - int supplement_undo_actions_if_exist(ObTxData *tx_data); + int supplement_tx_op_if_exist(ObTxData *tx_data); int prepare_for_safe_destroy(); diff --git a/src/storage/tx_table/ob_tx_table_interface.cpp b/src/storage/tx_table/ob_tx_table_interface.cpp index 70e21c5367..8c39a26ff8 100644 --- a/src/storage/tx_table/ob_tx_table_interface.cpp +++ b/src/storage/tx_table/ob_tx_table_interface.cpp @@ -57,6 +57,17 @@ int ObTxTableGuard::check_row_locked(const transaction::ObTransID &read_tx_id, } } +int ObTxTableGuard::load_tx_op(const transaction::ObTransID &tx_id, ObTxData &tx_data) +{ + if (OB_NOT_NULL(tx_table_)) { + ObReadTxDataArg arg(tx_id, epoch_, mini_cache_, false); + LoadTxOpFunctor functor(tx_data); + return tx_table_->check_with_tx_data(arg, functor); + } else { + return OB_NOT_INIT; + } +} + int ObTxTableGuard::check_sql_sequence_can_read(const transaction::ObTransID tx_id, const transaction::ObTxSEQ &sql_sequence, bool &can_read) diff --git a/src/storage/tx_table/ob_tx_table_interface.h b/src/storage/tx_table/ob_tx_table_interface.h index 441dae335e..7830769ce1 100644 --- a/src/storage/tx_table/ob_tx_table_interface.h +++ b/src/storage/tx_table/ob_tx_table_interface.h @@ -70,6 +70,8 @@ public: // dalegate functions const transaction::ObTxSEQ &sql_sequence, storage::ObStoreRowLockState &lock_state); + int load_tx_op(const transaction::ObTransID &tx_id, ObTxData &tx_data); + int check_sql_sequence_can_read(const transaction::ObTransID tx_id, const transaction::ObTxSEQ &sql_sequence, bool &can_read); int get_tx_state_with_scn(const transaction::ObTransID tx_id, diff --git a/src/storage/tx_table/ob_tx_table_iterator.cpp b/src/storage/tx_table/ob_tx_table_iterator.cpp index 87def4b7e4..57415ba956 100644 --- a/src/storage/tx_table/ob_tx_table_iterator.cpp +++ b/src/storage/tx_table/ob_tx_table_iterator.cpp @@ -73,6 +73,10 @@ int ObTxDataMemtableScanIterator::TxData2DatumRowConverter::init(ObTxData *tx_da ret = OB_INVALID_ARGUMENT; STORAGE_LOG(ERROR, "tx data is null", KR(ret)); } else if (INT64_MAX != tx_data->tx_id_.get_id()) {// normal tx data need local buffer to serialize + SpinRLockManualGuard tx_op_guard; + if (tx_data->op_guard_.is_valid()) { + tx_op_guard.lock(tx_data->op_guard_->get_lock()); + } buffer_len_ = tx_data->get_serialize_size(); if (nullptr == (serialize_buffer_ = (char *)DEFAULT_TX_DATA_ALLOCATOR.alloc(buffer_len_))) { ret = OB_ALLOCATE_MEMORY_FAILED; diff --git a/tools/deploy/mysql_test/test_suite/meta_info/r/mysql/meta_build_in_func_test.result b/tools/deploy/mysql_test/test_suite/meta_info/r/mysql/meta_build_in_func_test.result index 82408240a8..e491fb5c08 100644 --- a/tools/deploy/mysql_test/test_suite/meta_info/r/mysql/meta_build_in_func_test.result +++ b/tools/deploy/mysql_test/test_suite/meta_info/r/mysql/meta_build_in_func_test.result @@ -2835,22 +2835,22 @@ def test t1 t1 a a 8 20 1 Y 32768 0 63 +------+ select !w, !!w, !(!w), ! !w, not w, not not w, w is true, w is not false, (not w) is false, not(w is false), if(w,'true','false'), !1+1, !'a', !false, !isnull('a'), !!isnull('a') from (select 0 w union select 1) w; Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr -def !w 8 1 1 Y 32896 0 63 -def !!w 8 1 1 Y 32896 0 63 -def !(!w) 8 1 1 Y 32896 0 63 -def ! !w 8 1 1 Y 32896 0 63 -def not w 8 1 1 Y 32896 0 63 -def not not w 8 1 1 Y 32896 0 63 +def !w 3 1 1 Y 32896 0 63 +def !!w 3 1 1 Y 32896 0 63 +def !(!w) 3 1 1 Y 32896 0 63 +def ! !w 3 1 1 Y 32896 0 63 +def not w 3 1 1 Y 32896 0 63 +def not not w 3 1 1 Y 32896 0 63 def w is true 3 1 1 N 32897 0 63 def w is not false 3 1 1 N 32897 0 63 def (not w) is false 3 1 1 N 32897 0 63 -def not(w is false) 8 1 1 N 32897 0 63 +def not(w is false) 3 1 1 N 32897 0 63 def if(w,'true','false') 253 20 5 N 1 0 45 def !1+1 8 2 1 N 32897 0 63 -def !'a' 8 1 1 N 32897 0 63 -def !false 8 1 1 N 32897 0 63 -def !isnull('a') 8 1 1 N 32897 0 63 -def !!isnull('a') 8 1 1 N 32897 0 63 +def !'a' 3 1 1 N 32897 0 63 +def !false 3 1 1 N 32897 0 63 +def !isnull('a') 3 1 1 N 32897 0 63 +def !!isnull('a') 3 1 1 N 32897 0 63 +------+------+-------+------+-------+-----------+-----------+----------------+------------------+-----------------+----------------------+------+------+--------+--------------+---------------+ | !w | !!w | !(!w) | ! !w | not w | not not w | w is true | w is not false | (not w) is false | not(w is false) | if(w,'true','false') | !1+1 | !'a' | !false | !isnull('a') | !!isnull('a') | +------+------+-------+------+-------+-----------+-----------+----------------+------------------+-----------------+----------------------+------+------+--------+--------------+---------------+ diff --git a/tools/deploy/mysql_test/test_suite/px/r/mysql/alloc_material_for_producer_consumer_schedule_mode.result b/tools/deploy/mysql_test/test_suite/px/r/mysql/alloc_material_for_producer_consumer_schedule_mode.result index e423b4e270..4daa7ec65b 100644 --- a/tools/deploy/mysql_test/test_suite/px/r/mysql/alloc_material_for_producer_consumer_schedule_mode.result +++ b/tools/deploy/mysql_test/test_suite/px/r/mysql/alloc_material_for_producer_consumer_schedule_mode.result @@ -8,10 +8,10 @@ Query Plan ==================================================================================== |ID|OPERATOR |NAME |EST.ROWS|EST.TIME(us)| ------------------------------------------------------------------------------------ -|0 |TEMP TABLE TRANSFORMATION | |1 |5 | -|1 |├─PX COORDINATOR | |1 |3 | -|2 |│ └─EXCHANGE OUT DISTR |:EX10001 |1 |3 | -|3 |│ └─TEMP TABLE INSERT |TEMP1 |1 |3 | +|0 |TEMP TABLE TRANSFORMATION | |1 |4 | +|1 |├─PX COORDINATOR | |0 |3 | +|2 |│ └─EXCHANGE OUT DISTR |:EX10001 |0 |3 | +|3 |│ └─TEMP TABLE INSERT |TEMP1 |0 |3 | |4 |│ └─HASH GROUP BY | |1 |3 | |5 |│ └─EXCHANGE IN DISTR | |1 |3 | |6 |│ └─EXCHANGE OUT DISTR (HASH) |:EX10000 |1 |2 | diff --git a/tools/upgrade/do_upgrade_post.py b/tools/upgrade/do_upgrade_post.py index bb738ee5ab..53bda07e26 100755 --- a/tools/upgrade/do_upgrade_post.py +++ b/tools/upgrade/do_upgrade_post.py @@ -74,7 +74,7 @@ def do_upgrade(my_host, my_port, my_user, my_passwd, timeout, my_module_set, upg if run_modules.MODULE_HEALTH_CHECK in my_module_set: logging.info('================begin to run health check action ===============') - upgrade_health_checker.do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout, False) # need_check_major_status = False + upgrade_health_checker.do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout) logging.info('================succeed to run health check action ===============') if run_modules.MODULE_END_ROLLING_UPGRADE in my_module_set: diff --git a/tools/upgrade/do_upgrade_pre.py b/tools/upgrade/do_upgrade_pre.py index 8b8bd60294..4ab86e18c8 100755 --- a/tools/upgrade/do_upgrade_pre.py +++ b/tools/upgrade/do_upgrade_pre.py @@ -96,7 +96,7 @@ def do_upgrade(my_host, my_port, my_user, my_passwd, timeout, my_module_set, upg if run_modules.MODULE_HEALTH_CHECK in my_module_set: logging.info('================begin to run health check action ===============') - upgrade_health_checker.do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout, True) # need_check_major_status = True + upgrade_health_checker.do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout) logging.info('================succeed to run health check action ===============') except Exception, e: diff --git a/tools/upgrade/special_upgrade_action_pre.py b/tools/upgrade/special_upgrade_action_pre.py index d8e68be6c9..1cd519e6dd 100755 --- a/tools/upgrade/special_upgrade_action_pre.py +++ b/tools/upgrade/special_upgrade_action_pre.py @@ -28,14 +28,6 @@ def do_special_upgrade(conn, cur, timeout, user, passwd): # when upgrade across version, disable enable_ddl/major_freeze if current_version != target_version: actions.set_parameter(cur, 'enable_ddl', 'False', timeout) - actions.set_parameter(cur, 'enable_major_freeze', 'False', timeout) - actions.set_tenant_parameter(cur, '_enable_adaptive_compaction', 'False', timeout) - # wait scheduler in storage to notice adaptive_compaction is switched to false - time.sleep(60 * 2) - query_cur = actions.QueryCursor(cur) - wait_major_timeout = 600 - upgrade_health_checker.check_major_merge(query_cur, wait_major_timeout) - actions.do_suspend_merge(cur, timeout) # When upgrading from a version prior to 4.2 to version 4.2, the bloom_filter should be disabled. # The param _bloom_filter_enabled is no longer in use as of version 4.2, there is no need to enable it again. if actions.get_version(current_version) < actions.get_version('4.2.0.0')\ diff --git a/tools/upgrade/upgrade_checker.py b/tools/upgrade/upgrade_checker.py index 31332086c6..f2366634c8 100755 --- a/tools/upgrade/upgrade_checker.py +++ b/tools/upgrade/upgrade_checker.py @@ -411,9 +411,6 @@ def check_cluster_status(query_cur): (desc, results) = query_cur.exec_query("""select count(1) from CDB_OB_MAJOR_COMPACTION where (GLOBAL_BROADCAST_SCN > LAST_SCN or STATUS != 'IDLE')""") if results[0][0] > 0 : fail_list.append('{0} tenant is merging, please check'.format(results[0][0])) - (desc, results) = query_cur.exec_query("""select /*+ query_timeout(1000000000) */ count(1) from __all_virtual_tablet_compaction_info where max_received_scn > finished_scn and max_received_scn > 0""") - if results[0][0] > 0 : - fail_list.append('{0} tablet is merging, please check'.format(results[0][0])) logging.info('check cluster status success') # 5. 检查是否有异常租户(creating，延迟删除，恢复中) diff --git a/tools/upgrade/upgrade_health_checker.py b/tools/upgrade/upgrade_health_checker.py index 1c41fa068d..1c84427722 100755 --- a/tools/upgrade/upgrade_health_checker.py +++ b/tools/upgrade/upgrade_health_checker.py @@ -393,7 +393,7 @@ def check_until_timeout(query_cur, sql, value, timeout): time.sleep(10) # 开始健康检查 -def do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout, need_check_major_status, zone = ''): +def do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout, zone = ''): try: conn = mysql.connector.connect(user = my_user, password = my_passwd, @@ -410,8 +410,6 @@ def do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout, need check_paxos_replica(query_cur, timeout) check_schema_status(query_cur, timeout) check_server_version_by_zone(query_cur, zone) - if True == need_check_major_status: - check_major_merge(query_cur, timeout) except Exception, e: logging.exception('run error') raise e @@ -446,7 +444,7 @@ if __name__ == '__main__': zone = get_opt_zone() logging.info('parameters from cmd: host=\"%s\", port=%s, user=\"%s\", password=\"%s\", log-file=\"%s\", timeout=%s, zone=\"%s\"', \ host, port, user, password.replace('"', '\\"'), log_filename, timeout, zone) - do_check(host, port, user, password, upgrade_params, timeout, False, zone) # need_check_major_status = False + do_check(host, port, user, password, upgrade_params, timeout, zone) except mysql.connector.Error, e: logging.exception('mysql connctor error') raise e diff --git a/tools/upgrade/upgrade_post.py b/tools/upgrade/upgrade_post.py index a17c1673bc..cd85639744 100755 --- a/tools/upgrade/upgrade_post.py +++ b/tools/upgrade/upgrade_post.py @@ -674,7 +674,7 @@ # # if run_modules.MODULE_HEALTH_CHECK in my_module_set: # logging.info('================begin to run health check action ===============') -# upgrade_health_checker.do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout, False) # need_check_major_status = False +# upgrade_health_checker.do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout) # logging.info('================succeed to run health check action ===============') # # if run_modules.MODULE_END_ROLLING_UPGRADE in my_module_set: @@ -869,7 +869,7 @@ # # if run_modules.MODULE_HEALTH_CHECK in my_module_set: # logging.info('================begin to run health check action ===============') -# upgrade_health_checker.do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout, True) # need_check_major_status = True +# upgrade_health_checker.do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout) # logging.info('================succeed to run health check action ===============') # # except Exception, e: @@ -1355,14 +1355,6 @@ # # when upgrade across version, disable enable_ddl/major_freeze # if current_version != target_version: # actions.set_parameter(cur, 'enable_ddl', 'False', timeout) -# actions.set_parameter(cur, 'enable_major_freeze', 'False', timeout) -# actions.set_tenant_parameter(cur, '_enable_adaptive_compaction', 'False', timeout) -# # wait scheduler in storage to notice adaptive_compaction is switched to false -# time.sleep(60 * 2) -# query_cur = actions.QueryCursor(cur) -# wait_major_timeout = 600 -# upgrade_health_checker.check_major_merge(query_cur, wait_major_timeout) -# actions.do_suspend_merge(cur, timeout) # # When upgrading from a version prior to 4.2 to version 4.2, the bloom_filter should be disabled. # # The param _bloom_filter_enabled is no longer in use as of version 4.2, there is no need to enable it again. # if actions.get_version(current_version) < actions.get_version('4.2.0.0')\ @@ -2071,9 +2063,6 @@ # (desc, results) = query_cur.exec_query("""select count(1) from CDB_OB_MAJOR_COMPACTION where (GLOBAL_BROADCAST_SCN > LAST_SCN or STATUS != 'IDLE')""") # if results[0][0] > 0 : # fail_list.append('{0} tenant is merging, please check'.format(results[0][0])) -# (desc, results) = query_cur.exec_query("""select /*+ query_timeout(1000000000) */ count(1) from __all_virtual_tablet_compaction_info where max_received_scn > finished_scn and max_received_scn > 0""") -# if results[0][0] > 0 : -# fail_list.append('{0} tablet is merging, please check'.format(results[0][0])) # logging.info('check cluster status success') # ## 5. 检查是否有异常租户(creating，延迟删除，恢复中) @@ -2869,7 +2858,7 @@ # time.sleep(10) # ## 开始健康检查 -#def do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout, need_check_major_status, zone = ''): +#def do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout, zone = ''): # try: # conn = mysql.connector.connect(user = my_user, # password = my_passwd, @@ -2886,8 +2875,6 @@ # check_paxos_replica(query_cur, timeout) # check_schema_status(query_cur, timeout) # check_server_version_by_zone(query_cur, zone) -# if True == need_check_major_status: -# check_major_merge(query_cur, timeout) # except Exception, e: # logging.exception('run error') # raise e @@ -2922,7 +2909,7 @@ # zone = get_opt_zone() # logging.info('parameters from cmd: host=\"%s\", port=%s, user=\"%s\", password=\"%s\", log-file=\"%s\", timeout=%s, zone=\"%s\"', \ # host, port, user, password.replace('"', '\\"'), log_filename, timeout, zone) -# do_check(host, port, user, password, upgrade_params, timeout, False, zone) # need_check_major_status = False +# do_check(host, port, user, password, upgrade_params, timeout, zone) # except mysql.connector.Error, e: # logging.exception('mysql connctor error') # raise e @@ -3064,7 +3051,6 @@ # enable_ddl(cur, timeout) # enable_rebalance(cur, timeout) # enable_rereplication(cur, timeout) -# enable_major_freeze(cur, timeout) # except Exception, e: # logging.exception('run error') # raise e diff --git a/tools/upgrade/upgrade_post_checker.py b/tools/upgrade/upgrade_post_checker.py index 21175cb60d..77e18b1d36 100755 --- a/tools/upgrade/upgrade_post_checker.py +++ b/tools/upgrade/upgrade_post_checker.py @@ -130,7 +130,6 @@ def do_check(conn, cur, query_cur, timeout): enable_ddl(cur, timeout) enable_rebalance(cur, timeout) enable_rereplication(cur, timeout) - enable_major_freeze(cur, timeout) except Exception, e: logging.exception('run error') raise e diff --git a/tools/upgrade/upgrade_pre.py b/tools/upgrade/upgrade_pre.py index 5786d475c4..b781585fc8 100755 --- a/tools/upgrade/upgrade_pre.py +++ b/tools/upgrade/upgrade_pre.py @@ -674,7 +674,7 @@ # # if run_modules.MODULE_HEALTH_CHECK in my_module_set: # logging.info('================begin to run health check action ===============') -# upgrade_health_checker.do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout, False) # need_check_major_status = False +# upgrade_health_checker.do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout) # logging.info('================succeed to run health check action ===============') # # if run_modules.MODULE_END_ROLLING_UPGRADE in my_module_set: @@ -869,7 +869,7 @@ # # if run_modules.MODULE_HEALTH_CHECK in my_module_set: # logging.info('================begin to run health check action ===============') -# upgrade_health_checker.do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout, True) # need_check_major_status = True +# upgrade_health_checker.do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout) # logging.info('================succeed to run health check action ===============') # # except Exception, e: @@ -1355,14 +1355,6 @@ # # when upgrade across version, disable enable_ddl/major_freeze # if current_version != target_version: # actions.set_parameter(cur, 'enable_ddl', 'False', timeout) -# actions.set_parameter(cur, 'enable_major_freeze', 'False', timeout) -# actions.set_tenant_parameter(cur, '_enable_adaptive_compaction', 'False', timeout) -# # wait scheduler in storage to notice adaptive_compaction is switched to false -# time.sleep(60 * 2) -# query_cur = actions.QueryCursor(cur) -# wait_major_timeout = 600 -# upgrade_health_checker.check_major_merge(query_cur, wait_major_timeout) -# actions.do_suspend_merge(cur, timeout) # # When upgrading from a version prior to 4.2 to version 4.2, the bloom_filter should be disabled. # # The param _bloom_filter_enabled is no longer in use as of version 4.2, there is no need to enable it again. # if actions.get_version(current_version) < actions.get_version('4.2.0.0')\ @@ -2071,9 +2063,6 @@ # (desc, results) = query_cur.exec_query("""select count(1) from CDB_OB_MAJOR_COMPACTION where (GLOBAL_BROADCAST_SCN > LAST_SCN or STATUS != 'IDLE')""") # if results[0][0] > 0 : # fail_list.append('{0} tenant is merging, please check'.format(results[0][0])) -# (desc, results) = query_cur.exec_query("""select /*+ query_timeout(1000000000) */ count(1) from __all_virtual_tablet_compaction_info where max_received_scn > finished_scn and max_received_scn > 0""") -# if results[0][0] > 0 : -# fail_list.append('{0} tablet is merging, please check'.format(results[0][0])) # logging.info('check cluster status success') # ## 5. 检查是否有异常租户(creating，延迟删除，恢复中) @@ -2869,7 +2858,7 @@ # time.sleep(10) # ## 开始健康检查 -#def do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout, need_check_major_status, zone = ''): +#def do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout, zone = ''): # try: # conn = mysql.connector.connect(user = my_user, # password = my_passwd, @@ -2886,8 +2875,6 @@ # check_paxos_replica(query_cur, timeout) # check_schema_status(query_cur, timeout) # check_server_version_by_zone(query_cur, zone) -# if True == need_check_major_status: -# check_major_merge(query_cur, timeout) # except Exception, e: # logging.exception('run error') # raise e @@ -2922,7 +2909,7 @@ # zone = get_opt_zone() # logging.info('parameters from cmd: host=\"%s\", port=%s, user=\"%s\", password=\"%s\", log-file=\"%s\", timeout=%s, zone=\"%s\"', \ # host, port, user, password.replace('"', '\\"'), log_filename, timeout, zone) -# do_check(host, port, user, password, upgrade_params, timeout, False, zone) # need_check_major_status = False +# do_check(host, port, user, password, upgrade_params, timeout, zone) # except mysql.connector.Error, e: # logging.exception('mysql connctor error') # raise e @@ -3064,7 +3051,6 @@ # enable_ddl(cur, timeout) # enable_rebalance(cur, timeout) # enable_rereplication(cur, timeout) -# enable_major_freeze(cur, timeout) # except Exception, e: # logging.exception('run error') # raise e diff --git a/unittest/share/CMakeLists.txt b/unittest/share/CMakeLists.txt index b66b070ac8..206b885d46 100644 --- a/unittest/share/CMakeLists.txt +++ b/unittest/share/CMakeLists.txt @@ -103,3 +103,4 @@ add_subdirectory(location_cache) add_subdirectory(detect) add_subdirectory(vector) add_subdirectory(index_usage) +add_subdirectory(compaction) diff --git a/unittest/share/compaction/CMakeLists.txt b/unittest/share/compaction/CMakeLists.txt new file mode 100644 index 0000000000..a1cd90d178 --- /dev/null +++ b/unittest/share/compaction/CMakeLists.txt @@ -0,0 +1 @@ +storage_unittest(test_compaction_time_guard) \ No newline at end of file diff --git a/unittest/share/compaction/test_compaction_time_guard.cpp b/unittest/share/compaction/test_compaction_time_guard.cpp new file mode 100644 index 0000000000..657f07d217 --- /dev/null +++ b/unittest/share/compaction/test_compaction_time_guard.cpp @@ -0,0 +1,160 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#define USING_LOG_PREFIX STORAGE +#define private public +#define protected public + +#include "share/compaction/ob_compaction_time_guard.h" + +namespace oceanbase +{ +namespace unittest +{ + +using namespace compaction; + +class TestCompactionTimeGuard : public ::testing::Test +{ +public: + TestCompactionTimeGuard() {} + virtual ~TestCompactionTimeGuard() {} +}; + +TEST_F(TestCompactionTimeGuard, basic_time_guard) +{ + const uint16_t CAPACITY = ObCompactionTimeGuard::CAPACITY; + const uint64_t current_time = common::ObTimeUtility::current_time(); + ObCompactionTimeGuard time_guard; + // construction + ASSERT_EQ(CAPACITY, time_guard.capacity_); + ASSERT_EQ(0, time_guard.size_); + ASSERT_TRUE(time_guard.is_empty()); + time_guard.set_last_click_ts(current_time); + ASSERT_EQ(current_time, time_guard.add_time_); + ASSERT_EQ(current_time, time_guard.last_click_ts_); + + // click + ASSERT_TRUE(time_guard.click(CAPACITY)); + ASSERT_TRUE(time_guard.is_empty()); + for (uint16_t i = 0; i < CAPACITY; ++i) { + ASSERT_TRUE(time_guard.click(i)); + ASSERT_EQ(time_guard.size_, i + 1); + ASSERT_EQ(time_guard.event_times_[i], time_guard.get_specified_cost_time(i)); + } + ASSERT_TRUE(time_guard.click(CAPACITY + 1)); + ASSERT_EQ(CAPACITY, time_guard.size_); + + // add and assignment + ObCompactionTimeGuard other_guard; + ASSERT_EQ(CAPACITY, other_guard.capacity_); + ASSERT_TRUE(other_guard.is_empty()); + for (uint16_t i = 0; i < CAPACITY; ++i) { + ASSERT_EQ(0, other_guard.event_times_[i]); + } + other_guard = time_guard; + ASSERT_EQ(time_guard.guard_type_, other_guard.guard_type_); + ASSERT_EQ(time_guard.warn_threshold_, other_guard.warn_threshold_); + ASSERT_EQ(time_guard.capacity_, other_guard.capacity_); + ASSERT_EQ(time_guard.size_, other_guard.size_); + ASSERT_EQ(time_guard.last_click_ts_, other_guard.last_click_ts_); + ASSERT_EQ(time_guard.add_time_, other_guard.add_time_); + for (uint16_t i = 0; i < CAPACITY; ++i) { + ASSERT_EQ(time_guard.event_times_[i], other_guard.event_times_[i]); + } + + // add time guard + ObCompactionTimeGuard summary_guard; + for (uint16_t i = 0; i < 2 * CAPACITY; i++) { + summary_guard.add_time_guard(time_guard); + summary_guard.add_time_guard(other_guard); + } + for (uint16_t i = 0; i < CAPACITY; ++i) { + ASSERT_EQ(summary_guard.event_times_[i], time_guard.event_times_[i] * 4 * CAPACITY); + ASSERT_EQ(summary_guard.event_times_[i], other_guard.event_times_[i] * 4 * CAPACITY); + } + + // reuse + ObCompactionTimeGuard* guards[3] = {&time_guard, &other_guard, &summary_guard}; + for (int i = 0; i < 3; i++) { + guards[i]->reuse(); + ASSERT_TRUE(guards[i]->is_empty()); + ASSERT_EQ(CAPACITY, guards[i]->capacity_); + ASSERT_EQ(0, guards[i]->size_); + for (uint16_t j = 0; j < CAPACITY; ++j) { + ASSERT_EQ(0, guards[i]->event_times_[j]); + } + } + // out of order click and add time guard with diffrent size_ + time_guard.click(0); + ASSERT_EQ(1, time_guard.size_); + time_guard.click(2); + ASSERT_EQ(3, time_guard.size_); + time_guard.click(1); + ASSERT_EQ(3, time_guard.size_); + time_guard.click(10); + ASSERT_EQ(11, time_guard.size_); + other_guard.click(6); + ASSERT_EQ(7, other_guard.size_); + summary_guard.add_time_guard(other_guard); + ASSERT_EQ(summary_guard.size_, other_guard.size_); + summary_guard.add_time_guard(time_guard); + ASSERT_EQ(summary_guard.size_, time_guard.size_); +} + +TEST_F(TestCompactionTimeGuard, time_guard_to_string) +{ + ObStorageCompactionTimeGuard storage_guard; + for (uint16_t i = 0; i < ObStorageCompactionTimeGuard::COMPACTION_EVENT_MAX; i++) { + storage_guard.click(i); + storage_guard.event_times_[i] += ObStorageCompactionTimeGuard::COMPACTION_SHOW_TIME_THRESHOLD; + ASSERT_EQ(i + 1, storage_guard.size_); + } + ASSERT_EQ(ObStorageCompactionTimeGuard::COMPACTION_EVENT_MAX, storage_guard.size_); + storage_guard.event_times_[ObStorageCompactionTimeGuard::DAG_WAIT_TO_SCHEDULE] += 2 * ObStorageCompactionTimeGuard::COMPACTION_SHOW_TIME_THRESHOLD; + storage_guard.event_times_[ObStorageCompactionTimeGuard::DAG_FINISH] += 2 * ObCompactionTimeGuard::WARN_THRESHOLD; + STORAGE_LOG(INFO, "storage guard is", K(storage_guard)); + + ObRSCompactionTimeGuard rs_guard; + for (uint16_t i = 0; i < ObRSCompactionTimeGuard::COMPACTION_EVENT_MAX; i++) { + rs_guard.click(i); + rs_guard.event_times_[i] += 10 * i; + ASSERT_EQ(i + 1, rs_guard.size_); + } + ASSERT_EQ(ObRSCompactionTimeGuard::COMPACTION_EVENT_MAX, rs_guard.size_); + rs_guard.event_times_[ObRSCompactionTimeGuard::CKM_VERIFICATION] += 2 * ObCompactionTimeGuard::WARN_THRESHOLD; + STORAGE_LOG(INFO, "rs guard is", K(rs_guard)); + + ObCompactionScheduleTimeGuard schedule_guard; + for (uint16_t i = 0; i < ObCompactionScheduleTimeGuard::COMPACTION_EVENT_MAX; i++) { + schedule_guard.click(i); + schedule_guard.event_times_[i] += 10 * i; + ASSERT_EQ(i + 1, schedule_guard.size_); + } + ASSERT_EQ(ObCompactionScheduleTimeGuard::COMPACTION_EVENT_MAX, schedule_guard.size_); + schedule_guard.event_times_[ObCompactionScheduleTimeGuard::SCHEDULER_NEXT_ROUND] += 2 * ObCompactionTimeGuard::WARN_THRESHOLD; + STORAGE_LOG(INFO, "schedule guard is", K(schedule_guard)); +} + +} // namespace unittest +} // namespace oceanbase + +int main(int argc, char **argv) +{ + ::testing::InitGoogleTest(&argc, argv); + system("rm -f test_compaction_time_guard.log*"); + OB_LOGGER.set_file_name("test_compaction_time_guard.log"); + OB_LOGGER.set_log_level("INFO"); + OB_LOGGER.set_max_file_size(256*1024*1024); + return RUN_ALL_TESTS(); +} diff --git a/unittest/sql/resolver/expr/test_raw_expr_resolver.result b/unittest/sql/resolver/expr/test_raw_expr_resolver.result index df19f4e1ac..5e1ea84d4d 100644 --- a/unittest/sql/resolver/expr/test_raw_expr_resolver.result +++ b/unittest/sql/resolver/expr/test_raw_expr_resolver.result @@ -1772,7 +1772,7 @@ "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } [13] max(c1) @@ -1865,7 +1865,7 @@ "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } [14] c1 in (select t1 from table1) diff --git a/unittest/sql/rewrite/result/test_transformer_stmt_after_el.result b/unittest/sql/rewrite/result/test_transformer_stmt_after_el.result index 9d44114400..08fa9d5172 100644 --- a/unittest/sql/rewrite/result/test_transformer_stmt_after_el.result +++ b/unittest/sql/rewrite/result/test_transformer_stmt_after_el.result @@ -73219,7 +73219,7 @@ SQL: select sum(c1), count(*) from t1 group by c2 order by c2 limit 100; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -73280,7 +73280,7 @@ SQL: select sum(c1), count(*) from t1 group by c2 order by c2 limit 100; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -73466,7 +73466,7 @@ SQL: select sum(c1), count(*) from t1 group by c2 order by c2 limit 100; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -73512,7 +73512,7 @@ SQL: select sum(c1), count(*) from t1 group by c2 order by c2 limit 100; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], diff --git a/unittest/sql/rewrite/result/test_transformer_stmt_after_query_push_down.result b/unittest/sql/rewrite/result/test_transformer_stmt_after_query_push_down.result index 3294722554..b8a690d5f0 100644 --- a/unittest/sql/rewrite/result/test_transformer_stmt_after_query_push_down.result +++ b/unittest/sql/rewrite/result/test_transformer_stmt_after_query_push_down.result @@ -6742,7 +6742,7 @@ SQL: select count(*) from (select * from t1) a group by a.c2 limit 1, 10; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null ======= "is_lob_column":false, @@ -6930,7 +6930,7 @@ SQL: select count(*) from (select * from t1) a group by a.c2 limit 1, 10; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -7703,7 +7703,7 @@ SQL: select a.c2 from (select * from t1) a; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -7839,7 +7839,7 @@ SQL: select a.c2 from (select * from t1) a; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], diff --git a/unittest/sql/rewrite/result/test_transformer_stmt_after_together.result b/unittest/sql/rewrite/result/test_transformer_stmt_after_together.result index 79e93bbc3e..e4aa9ba9af 100644 --- a/unittest/sql/rewrite/result/test_transformer_stmt_after_together.result +++ b/unittest/sql/rewrite/result/test_transformer_stmt_after_together.result @@ -78863,7 +78863,7 @@ SQL: select * from t1, t2 where t1.c1 > any (select c2 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -79001,7 +79001,7 @@ SQL: select * from t1, t2 where t1.c1 > any (select c2 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -96135,7 +96135,7 @@ SQL: select max(k) from (select c1 * c2 as k from t2 where c2 in (select c2 from "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -96353,7 +96353,7 @@ SQL: select max(k) from (select c1 * c2 as k from t2 where c2 in (select c2 from "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -97032,7 +97032,7 @@ SQL: select max(c1 * c2) from t2 where c2 in (select c2 from t3); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -97250,7 +97250,7 @@ SQL: select max(c1 * c2) from t2 where c2 in (select c2 from t3); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -148454,7 +148454,7 @@ SQL: select * from t1 where c1 >ANY (select c1 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -148592,7 +148592,7 @@ SQL: select * from t1 where c1 >ANY (select c1 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -149862,7 +149862,7 @@ SQL: select * from t1 where c1 >>>>>> de0eba8f08f... GIS sql syntax @@ -155209,7 +155209,7 @@ SQL: select * from t1 where Exists (select c1 from t2 where c2 >ALL (select c1 f "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -155377,7 +155377,7 @@ SQL: select * from t1 where Exists (select c1 from t2 where c2 >ALL (select c1 f "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -155539,7 +155539,7 @@ SQL: select * from t1 where Exists (select c1 from t2 where c2 >ALL (select c1 f "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], diff --git a/unittest/sql/rewrite/result/test_transformer_stmt_after_trans_aggr.result b/unittest/sql/rewrite/result/test_transformer_stmt_after_trans_aggr.result index 7742bec8fb..2676b59be3 100644 --- a/unittest/sql/rewrite/result/test_transformer_stmt_after_trans_aggr.result +++ b/unittest/sql/rewrite/result/test_transformer_stmt_after_trans_aggr.result @@ -200,7 +200,7 @@ SQL: select max(c1) from t1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -335,7 +335,7 @@ SQL: select max(c1) from t1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -880,7 +880,7 @@ SQL: select min(c1) from t1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -1015,7 +1015,7 @@ SQL: select min(c1) from t1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -1560,7 +1560,7 @@ SQL: select max(c1) as max from t1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":true, @@ -1695,7 +1695,7 @@ SQL: select max(c1) as max from t1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -2240,7 +2240,7 @@ SQL: select min(c1) as min from t1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":true, @@ -2375,7 +2375,7 @@ SQL: select min(c1) as min from t1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -2920,7 +2920,7 @@ SQL: select max(c2) from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -3055,7 +3055,7 @@ SQL: select max(c2) from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -3931,7 +3931,7 @@ SQL: select max(c3) from t10; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -4066,7 +4066,7 @@ SQL: select max(c3) from t10; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -5810,7 +5810,7 @@ SQL: select * from t9 where c2 = (select max(c2) from t9); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -5945,7 +5945,7 @@ SQL: select * from t9 where c2 = (select max(c2) from t9); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -6825,7 +6825,7 @@ SQL: select max(c1) from t1 as tmp; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -6960,7 +6960,7 @@ SQL: select max(c1) from t1 as tmp; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -7953,7 +7953,7 @@ SQL: select c1 from t1 where c1 > (select max(c1) from t1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -8088,7 +8088,7 @@ SQL: select c1 from t1 where c1 > (select max(c1) from t1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -8637,7 +8637,7 @@ SQL: select max(c1) as max from t1 having max(c1) > 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":true, @@ -8803,7 +8803,7 @@ SQL: select max(c1) as max from t1 having max(c1) > 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -8938,7 +8938,7 @@ SQL: select max(c1) as max from t1 having max(c1) > 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -9483,7 +9483,7 @@ SQL: select min(c1) as min from t1 having min(c1) > 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":true, @@ -9649,7 +9649,7 @@ SQL: select min(c1) as min from t1 having min(c1) > 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -9784,7 +9784,7 @@ SQL: select min(c1) as min from t1 having min(c1) > 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -10329,7 +10329,7 @@ SQL: select max(c1) as max from t1 having max(c1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":true, @@ -10494,7 +10494,7 @@ SQL: select max(c1) as max from t1 having max(c1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null ======= "is_lob_column":false, @@ -10606,7 +10606,7 @@ SQL: select max(c1) as max from t1 having max(c1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -11151,7 +11151,7 @@ SQL: select min(c1) as min from t1 having true; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":true, @@ -11316,7 +11316,7 @@ SQL: select min(c1) as min from t1 having true; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -11867,7 +11867,7 @@ SQL: select max(c1) from t7 having false; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -12032,7 +12032,7 @@ SQL: select max(c1) from t7 having false; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -12644,7 +12644,7 @@ SQL: select max(c1) from pt2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -12779,7 +12779,7 @@ SQL: select max(c1) from pt2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -13380,7 +13380,7 @@ SQL: select max(c1) as max from pt2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":true, @@ -13515,7 +13515,7 @@ SQL: select max(c1) as max from pt2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -14116,7 +14116,7 @@ SQL: select min(c1) from pt2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -14251,7 +14251,7 @@ SQL: select min(c1) from pt2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -14852,7 +14852,7 @@ SQL: select min(c1) as min from pt2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":true, @@ -14987,7 +14987,7 @@ SQL: select min(c1) as min from pt2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -15588,7 +15588,7 @@ SQL: select * from (select max(c1) from pt2) as tmp_table; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -15723,7 +15723,7 @@ SQL: select * from (select max(c1) from pt2) as tmp_table; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -17028,7 +17028,7 @@ SQL: select * from t2 where t2.c1 > (select max(c1) from pt2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -17163,7 +17163,7 @@ SQL: select * from t2 where t2.c1 > (select max(c1) from pt2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -17768,7 +17768,7 @@ SQL: select * from (select min(c1) as min from pt2) as tmp_table; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -17903,7 +17903,7 @@ SQL: select * from (select min(c1) as min from pt2) as tmp_table; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -19208,7 +19208,7 @@ SQL: select * from t2 where t2.c1 > (select min(c1) as min from pt2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":true, @@ -19343,7 +19343,7 @@ SQL: select * from t2 where t2.c1 > (select min(c1) as min from pt2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -19948,7 +19948,7 @@ SQL: select * from (select max(c1) from t1) as tmp_table; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -20083,7 +20083,7 @@ SQL: select * from (select max(c1) from t1) as tmp_table; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -21332,7 +21332,7 @@ SQL: select * from t2 where t2.c1 > (select max(c1) from t1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -21467,7 +21467,7 @@ SQL: select * from t2 where t2.c1 > (select max(c1) from t1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -22016,7 +22016,7 @@ SQL: select * from (select max(c1) as max from t1) as tmp_table; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -22151,7 +22151,7 @@ SQL: select * from (select max(c1) as max from t1) as tmp_table; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -23400,7 +23400,7 @@ SQL: select * from t2 where t2.c1 > (select max(c1) as max from t1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":true, @@ -23535,7 +23535,7 @@ SQL: select * from t2 where t2.c1 > (select max(c1) as max from t1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -24084,7 +24084,7 @@ SQL: select * from (select min(c1) from t1) as tmp_table; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -24219,7 +24219,7 @@ SQL: select * from (select min(c1) from t1) as tmp_table; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -25468,7 +25468,7 @@ SQL: select * from t2 where t2.c1 > (select min(c1) from t1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -25603,7 +25603,7 @@ SQL: select * from t2 where t2.c1 > (select min(c1) from t1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -26152,7 +26152,7 @@ SQL: select * from (select min(c1) as min from t1) as tmp_table; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -26287,7 +26287,7 @@ SQL: select * from (select min(c1) as min from t1) as tmp_table; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -27536,7 +27536,7 @@ SQL: select * from t2 where t2.c1 > (select min(c1) as min from t1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":true, @@ -27671,7 +27671,7 @@ SQL: select * from t2 where t2.c1 > (select min(c1) as min from t1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -28220,7 +28220,7 @@ SQL: select max(c1) from t7; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -28355,7 +28355,7 @@ SQL: select max(c1) from t7; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -28991,7 +28991,7 @@ SQL: select min(c1) from t7; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -29126,7 +29126,7 @@ SQL: select min(c1) from t7; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -29833,7 +29833,7 @@ SQL: select max(c1 + 1) + 1 from t1 limit 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -30070,7 +30070,7 @@ SQL: select max(c1 + 1) + 1 from t1 limit 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -30337,7 +30337,7 @@ SQL: select max(c1) from t1 limit 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -30472,7 +30472,7 @@ SQL: select max(c1) from t1 limit 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -31046,7 +31046,7 @@ SQL: select min(c1) from t1 limit 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -31181,7 +31181,7 @@ SQL: select min(c1) from t1 limit 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -31755,7 +31755,7 @@ SQL: select min(c1) as min from t1 limit 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":true, @@ -31890,7 +31890,7 @@ SQL: select min(c1) as min from t1 limit 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -32565,7 +32565,7 @@ SQL: select min(c1 + 1) + 1 from t1 limit 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -32802,7 +32802,7 @@ SQL: select min(c1 + 1) + 1 from t1 limit 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -33069,7 +33069,7 @@ SQL: select max(c1) as max from t1 limit 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":true, @@ -33204,7 +33204,7 @@ SQL: select max(c1) as max from t1 limit 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -33879,7 +33879,7 @@ SQL: select max(c1 + 1) + 1 as max from t1 limit 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -34116,7 +34116,7 @@ SQL: select max(c1 + 1) + 1 as max from t1 limit 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -34484,7 +34484,7 @@ SQL: select min(c1 + 1) + 1 as min from t1 limit 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -34721,7 +34721,7 @@ SQL: select min(c1 + 1) + 1 as min from t1 limit 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -34988,7 +34988,7 @@ SQL: select max(c1) as max from t1 having min(c1) > 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":true, @@ -35154,7 +35154,7 @@ SQL: select max(c1) as max from t1 having min(c1) > 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -35289,7 +35289,7 @@ SQL: select max(c1) as max from t1 having min(c1) > 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -35388,7 +35388,7 @@ SQL: select max(c1) as max from t1 having min(c1) > 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -35626,7 +35626,7 @@ SQL: select min(c1) as min from t1 having max(c1) > 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":true, @@ -35792,7 +35792,7 @@ SQL: select min(c1) as min from t1 having max(c1) > 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -35927,7 +35927,7 @@ SQL: select min(c1) as min from t1 having max(c1) > 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -36026,7 +36026,7 @@ SQL: select min(c1) as min from t1 having max(c1) > 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -36264,7 +36264,7 @@ SQL: select max(c1) from t1 having 1 in (select c1 from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -36557,7 +36557,7 @@ SQL: select max(c1) from t1 having 1 in (select c1 from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -38864,7 +38864,7 @@ SQL: select max(c4) from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -38999,7 +38999,7 @@ SQL: select max(c4) from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -39279,7 +39279,7 @@ SQL: select min(999) from t7 having max(c1) > 0; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -39445,7 +39445,7 @@ SQL: select min(999) from t7 having max(c1) > 0; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -39561,7 +39561,7 @@ SQL: select min(999) from t7 having max(c1) > 0; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -39660,7 +39660,7 @@ SQL: select min(999) from t7 having max(c1) > 0; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -40692,7 +40692,7 @@ SQL: select max(1+2) from t7 limit 10; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -40808,7 +40808,7 @@ SQL: select max(1+2) from t7 limit 10; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -41292,7 +41292,7 @@ SQL: select max(1+2) from t7; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -41408,7 +41408,7 @@ SQL: select max(1+2) from t7; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -41863,7 +41863,7 @@ SQL: select min(1+2) from t7 where c2 = 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -41979,7 +41979,7 @@ SQL: select min(1+2) from t7 where c2 = 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -42675,7 +42675,7 @@ SQL: select max(1+2), 3 from t7; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -42838,7 +42838,7 @@ SQL: select max(1+2), 3 from t7; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -42998,7 +42998,7 @@ SQL: select max(1+2) from t7,t8 where t7.c1 = t8.c1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -43114,7 +43114,7 @@ SQL: select max(1+2) from t7,t8 where t7.c1 = t8.c1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -43860,7 +43860,7 @@ SQL: select min(1+2) from t7 inner join t8 on t7.c1=t8.c1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -43976,7 +43976,7 @@ SQL: select min(1+2) from t7 inner join t8 on t7.c1=t8.c1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -44722,7 +44722,7 @@ SQL: select max(1+2) from t7 left join t8 using(c1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -44838,7 +44838,7 @@ SQL: select max(1+2) from t7 left join t8 using(c1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -49617,7 +49617,7 @@ SQL: select * from t7 where c1 in (select max(999+1) from t8); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -49733,7 +49733,7 @@ SQL: select * from t7 where c1 in (select max(999+1) from t8); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -50839,7 +50839,7 @@ SQL: select * from t7 where c1 in (select max(999+1) from t8 where t8.c1=2 and t "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -50955,7 +50955,7 @@ SQL: select * from t7 where c1 in (select max(999+1) from t8 where t8.c1=2 and t "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -52696,7 +52696,7 @@ SQL: select max(1) from t7 order by 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -52812,7 +52812,7 @@ SQL: select max(1) from t7 order by 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -52895,7 +52895,7 @@ SQL: select max(1) from t7 order by 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "ascending":"NULLs_first_asc" @@ -53405,7 +53405,7 @@ SQL: select max(1+2) from t7 order by c1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -53521,7 +53521,7 @@ SQL: select max(1+2) from t7 order by c1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -54171,7 +54171,7 @@ SQL: select max(1+2) from t7 order by c2,c3; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -54287,7 +54287,7 @@ SQL: select max(1+2) from t7 order by c2,c3; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -56649,7 +56649,7 @@ SQL: select c1 from t1 where c1 < some (select c1 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -58263,7 +58263,7 @@ SQL: select c1 from t1 where c1 < all (select c1 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -58439,7 +58439,7 @@ SQL: select c1 from t1 where c1 < all (select c1 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -59496,7 +59496,7 @@ SQL: select c1 from t1 where c1 in (select c1 from t2 where c2 >= some(select c1 "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -59664,7 +59664,7 @@ SQL: select c1 from t1 where c1 in (select c1 from t2 where c2 >= some(select c1 "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -59826,7 +59826,7 @@ SQL: select c1 from t1 where c1 in (select c1 from t2 where c2 >= some(select c1 "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -60907,7 +60907,7 @@ SQL: select c1 from t1 where c1 < all (select c1 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -61075,7 +61075,7 @@ SQL: select c1 from t1 where c1 < all (select c1 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -61237,7 +61237,7 @@ SQL: select c1 from t1 where c1 < all (select c1 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -62829,7 +62829,7 @@ SQL: select c1 from t1 where c1 in (select c1 from t2 where c2 >= some(select c1 "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -63081,7 +63081,7 @@ SQL: select c1 from t1 where c1 + 1 > any (select c1 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -64438,7 +64438,7 @@ SQL: select c1 from t1 where c1 > any (select c1 from t2 where c2 >= some(select "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -64711,7 +64711,7 @@ SQL: select c1 from t1 where c1 > any (select c1 from t2 where c2 >= some(select "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -65216,7 +65216,7 @@ SQL: select c1 from t1 where c1 > any (select c1 from t2 where c2 >= some(select "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -65394,7 +65394,7 @@ SQL: select c1 from t1 where c1 > any (select c1 from t2 where c2 >= some(select "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -66532,7 +66532,7 @@ SQL: select c1 from t1 group by c1 > any (select c1 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -66670,7 +66670,7 @@ SQL: select c1 from t1 group by c1 > any (select c1 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -67745,7 +67745,7 @@ SQL: select c1 from t1 having c1 < any (select c1 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -67883,7 +67883,7 @@ SQL: select c1 from t1 having c1 < any (select c1 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -68890,7 +68890,7 @@ SQL: select c1 from t1 order by c1 > all (select c1 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -69058,7 +69058,7 @@ SQL: select c1 from t1 order by c1 > all (select c1 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -69220,7 +69220,7 @@ SQL: select c1 from t1 order by c1 > all (select c1 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } >>>>>>> de0eba8f08f... GIS sql syntax @@ -70620,7 +70620,7 @@ SQL: select c1 from t1 order by c1 > all (select c1 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null >>>>>>> de0eba8f08f... GIS sql syntax }, @@ -71014,7 +71014,7 @@ SQL: select c1 from t1 where c1 > any (select c1 from t2 where c2 >= some(select "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -73708,7 +73708,7 @@ SQL: select c1 from t1 group by c1 > any (select c1 from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -73843,7 +73843,7 @@ SQL: select c1 from t1 group by c1 > any (select c1 from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -75263,7 +75263,7 @@ SQL: select c1 from t1 order by c1 > all (select c1 from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -75427,7 +75427,7 @@ SQL: select c1 from t1 order by c1 > all (select c1 from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -75621,7 +75621,7 @@ SQL: select c1 from t1 order by c1 > all (select c1 from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -76557,7 +76557,7 @@ SQL: select c1 from t1 order by c1 < all (select c1 from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -76722,7 +76722,7 @@ SQL: select c1 from t1 order by c1 < all (select c1 from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -76895,7 +76895,7 @@ SQL: select c1 from t1 order by c1 < all (select c1 from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null ======= 0 @@ -79596,7 +79596,7 @@ SQL: select c1 from t1 where c1 + (1 < any (select c1 from t2)); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -79731,7 +79731,7 @@ SQL: select c1 from t1 where c1 + (1 < any (select c1 from t2)); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -80774,7 +80774,7 @@ SQL: select c1 from t1 where c1 > any (select c2 from t9); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -80912,7 +80912,7 @@ SQL: select c1 from t1 where c1 > any (select c2 from t9); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -82374,7 +82374,7 @@ SQL: select max(c3) from t9 where c2 = 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -82509,7 +82509,7 @@ SQL: select max(c3) from t9 where c2 = 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -83717,7 +83717,7 @@ SQL: select max(c3) from t9 where c2 is TRUE; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -84163,7 +84163,7 @@ SQL: select max(c3) from t9 where 1 = c2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -84298,7 +84298,7 @@ SQL: select max(c3) from t9 where 1 = c2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -85488,7 +85488,7 @@ SQL: select max(c3) from t9 where c2 != 1; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -86018,7 +86018,7 @@ SQL: select max(c3) from t9 where c2 is TRUE; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -86366,7 +86366,7 @@ SQL: select max(c3) from t9 where c2 is TRUE; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -86701,11 +86701,11 @@ SQL: select max(c3) from t9 where c2 <1; "type":0 }, <<<<<<< HEAD - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null ======= "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -86959,7 +86959,7 @@ SQL: select max(c3) from t9 where c2 <1; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -87541,7 +87541,7 @@ SQL: select c1 from t1 where c1 > any (select c2 from t9); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -87778,7 +87778,7 @@ SQL: select max(c3) from t9 where c2 = 1 and c1 = 1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -87887,7 +87887,7 @@ SQL: select max(c3) from t9 where c2 = 1 and c1 = 1; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -88089,7 +88089,7 @@ SQL: select max(c3) from t9 where c2 = 1 and c1 = 1; "type":0 }, <<<<<<< HEAD - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -90141,7 +90141,7 @@ SQL: select c1 from t1 where c1 > any (select c1 from t2 having c1 > 1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null ======= "is_lob_column":false, @@ -90470,7 +90470,7 @@ SQL: select c1 from t1 where c1 > any (select c1 from t2 having c1 > 1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -90725,7 +90725,7 @@ SQL: select max(c3) from t9 where c2 >1; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null >>>>>>> de0eba8f08f... GIS sql syntax }, @@ -91551,7 +91551,7 @@ SQL: select max(c3) from t9 where c2 >1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -91802,7 +91802,7 @@ SQL: select max(c3) from t9 where c2 >1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -92574,7 +92574,7 @@ SQL: select max(c3) from t9 where c2 <1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -92686,7 +92686,7 @@ SQL: select max(c3) from t9 where c2 <1; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -93274,7 +93274,7 @@ SQL: select max(c3) from t9 where c2 <1; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -98484,7 +98484,7 @@ SQL: select c1 from t1 where c1 > all (select c2 from t9); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -98582,7 +98582,7 @@ SQL: select c1 from t1 where c1 > all (select c2 from t9); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ] @@ -100022,7 +100022,7 @@ SQL: SELECT MIN(1) FROM agg_t2 ORDER BY (SELECT AVG(c2) FROM agg_t1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -100138,7 +100138,7 @@ SQL: SELECT MIN(1) FROM agg_t2 ORDER BY (SELECT AVG(c2) FROM agg_t1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -100237,7 +100237,7 @@ SQL: SELECT MIN(1) FROM agg_t2 ORDER BY (SELECT AVG(c2) FROM agg_t1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -100339,7 +100339,7 @@ SQL: SELECT MIN(1) FROM agg_t2 ORDER BY (SELECT AVG(c2) FROM agg_t1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -100566,7 +100566,7 @@ SQL: SELECT MIN(1) FROM agg_t2 ORDER BY (SELECT AVG(c2) FROM agg_t1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -100701,7 +100701,7 @@ SQL: SELECT MIN(1) FROM agg_t2 ORDER BY (SELECT AVG(c2) FROM agg_t1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -101280,7 +101280,7 @@ SQL: SELECT MIN(1) FROM agg_t2 ORDER BY (SELECT AVG(c2) FROM agg_t1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -101415,7 +101415,7 @@ SQL: SELECT MIN(1) FROM agg_t2 ORDER BY (SELECT AVG(c2) FROM agg_t1); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -101889,7 +101889,7 @@ SQL: SELECT MIN(DISTINCT OUTR . `col_varchar_10`) AS X FROM B AS OUTR WHERE (OUT "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":true, @@ -102054,7 +102054,7 @@ SQL: SELECT MIN(DISTINCT OUTR . `col_varchar_10`) AS X FROM B AS OUTR WHERE (OUT "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -105178,7 +105178,7 @@ SQL: select 5, max(c2) from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -105313,7 +105313,7 @@ SQL: select 5, max(c2) from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -106185,7 +106185,7 @@ SQL: select max(c2), 5 from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -106367,7 +106367,7 @@ SQL: select max(c2), 5 from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -107286,7 +107286,7 @@ SQL: select 5, min(c2) from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -107421,7 +107421,7 @@ SQL: select 5, min(c2) from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -108293,7 +108293,7 @@ SQL: select min(c2), 5 from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -108475,7 +108475,7 @@ SQL: select min(c2), 5 from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -109441,7 +109441,7 @@ SQL: select 5, 5, min(c2), 7 + 5, floor(5) from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -109670,7 +109670,7 @@ SQL: select 5, 5, min(c2), 7 + 5, floor(5) from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -110601,7 +110601,7 @@ SQL: select abs(-55), max(c2), 3 + 2, 5 from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -110830,7 +110830,7 @@ SQL: select abs(-55), max(c2), 3 + 2, 5 from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -112006,7 +112006,7 @@ SQL: select 5, max(c2), c1 from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -112207,7 +112207,7 @@ SQL: select 5, max(c2), c1 from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -112756,7 +112756,7 @@ SQL: select c1, min(c2), 233333 from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -112938,7 +112938,7 @@ SQL: select c1, min(c2), 233333 from t9; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -113696,7 +113696,7 @@ SQL: select * from (select c1 as col from t1) as L where col = (select max(c1) f "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -113831,7 +113831,7 @@ SQL: select * from (select c1 as col from t1) as L where col = (select max(c1) f "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -116388,7 +116388,7 @@ SQL: select * from (select a.acctnbr as l1, a.timeuniqueextn as l2 from cb_loan "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -116900,7 +116900,7 @@ SQL: select * from (select a.acctnbr as l1, a.timeuniqueextn as l2 from cb_loan "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } <<<<<<< HEAD diff --git a/unittest/sql/rewrite/result/test_transformer_stmt_after_trans_simplify.result b/unittest/sql/rewrite/result/test_transformer_stmt_after_trans_simplify.result index 65b9340941..22d8af7c5b 100644 --- a/unittest/sql/rewrite/result/test_transformer_stmt_after_trans_simplify.result +++ b/unittest/sql/rewrite/result/test_transformer_stmt_after_trans_simplify.result @@ -25223,7 +25223,7 @@ SQL: select max(a), min(c) from pjt1,pjt2,pjt3; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -25336,7 +25336,7 @@ SQL: select max(a), min(c) from pjt1,pjt2,pjt3; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -25478,7 +25478,7 @@ SQL: select max(a), min(c) from pjt1,pjt2,pjt3; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -25576,7 +25576,7 @@ SQL: select max(a), min(c) from pjt1,pjt2,pjt3; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -26919,7 +26919,7 @@ SQL: select count(distinct g), max(d), min(a) from pjt1,pjt2,pjt3,pjt4; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -27033,7 +27033,7 @@ SQL: select count(distinct g), max(d), min(a) from pjt1,pjt2,pjt3,pjt4; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -27147,7 +27147,7 @@ SQL: select count(distinct g), max(d), min(a) from pjt1,pjt2,pjt3,pjt4; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -27297,7 +27297,7 @@ SQL: select count(distinct g), max(d), min(a) from pjt1,pjt2,pjt3,pjt4; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -27395,7 +27395,7 @@ SQL: select count(distinct g), max(d), min(a) from pjt1,pjt2,pjt3,pjt4; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -27493,7 +27493,7 @@ SQL: select count(distinct g), max(d), min(a) from pjt1,pjt2,pjt3,pjt4; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -28837,7 +28837,7 @@ SQL: select sum(distinct a) from pjt1,pjt2,pjt3 where b=d; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -29271,7 +29271,7 @@ SQL: select sum(distinct a) from pjt1,pjt2,pjt3 where b=d; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -30725,7 +30725,7 @@ SQL: select max(a) from pjt1,pjt2,pjt3,pjt4 where b=d; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -31007,7 +31007,7 @@ SQL: select max(a) from pjt1,pjt2,pjt3,pjt4 where b=d; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -32007,7 +32007,7 @@ SQL: select avg(distinct c) from pjt1,pjt2,pjt3,pjt4 where a=e; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -32157,7 +32157,7 @@ SQL: select avg(distinct c) from pjt1,pjt2,pjt3,pjt4 where a=e; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -32255,7 +32255,7 @@ SQL: select avg(distinct c) from pjt1,pjt2,pjt3,pjt4 where a=e; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -32353,7 +32353,7 @@ SQL: select avg(distinct c) from pjt1,pjt2,pjt3,pjt4 where a=e; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -32835,7 +32835,7 @@ SQL: select sum(distinct a) from pjt1,pjt2,pjt3 where b=d; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -32970,7 +32970,7 @@ SQL: select sum(distinct a) from pjt1,pjt2,pjt3 where b=d; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -33301,7 +33301,7 @@ SQL: select sum(distinct a) from pjt1,pjt2,pjt3 where b=d; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -33400,7 +33400,7 @@ SQL: select sum(distinct a) from pjt1,pjt2,pjt3 where b=d; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -34381,7 +34381,7 @@ SQL: select * from pjt1, (select min(d) as x from pjt2,pjt3) where a=x; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } >>>>>>> de0eba8f08f... GIS sql syntax @@ -35216,11 +35216,11 @@ SQL: select * from pjt1, (select min(d) as x from pjt2,pjt3) where a=x; "type":0 }, <<<<<<< HEAD - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null ======= "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ] @@ -35566,7 +35566,7 @@ SQL: select * from pjt1, (select min(d) as x from pjt2,pjt3) where a=x; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null ======= "is_lob_column":false, @@ -37368,7 +37368,7 @@ SQL: select * from pjt1, (select count(distinct h)as x from pjt2,pjt3,pjt4 where "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":true, @@ -37645,7 +37645,7 @@ SQL: select * from pjt1, (select count(distinct h)as x from pjt2,pjt3,pjt4 where "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -38205,7 +38205,7 @@ SQL: select min(b) from pjt1, (select count(distinct h)as x from pjt2,pjt3,pjt4 "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null ======= "is_lob_column":false, @@ -38822,7 +38822,7 @@ SQL: select min(b) from pjt1, (select count(distinct h)as x from pjt2,pjt3,pjt4 "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":true, @@ -39155,7 +39155,7 @@ SQL: select min(b) from pjt1, (select count(distinct h)as x from pjt2,pjt3,pjt4 "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -39937,7 +39937,7 @@ SQL: select min(b) from pjt1, (select count(distinct h)as x from pjt2,pjt3,pjt4 "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -40195,7 +40195,7 @@ SQL: select min(b) from pjt1, (select count(distinct h)as x from pjt2,pjt3,pjt4 "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -40595,7 +40595,7 @@ SQL: select min(b) from pjt1, (select count(distinct h)as x from pjt2,pjt3,pjt4 "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":true, @@ -40874,7 +40874,7 @@ SQL: select min(b) from pjt1, (select count(distinct h)as x from pjt2,pjt3,pjt4 "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -42296,7 +42296,7 @@ SQL: select distinct a from pjt1 where b>(select sum(distinct c) from pjt1,pjt2 "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -42553,7 +42553,7 @@ SQL: select distinct a from pjt1 where b>(select sum(distinct c) from pjt1,pjt2 "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -44218,7 +44218,7 @@ SQL: select distinct a from pjt1 where b>(select avg(distinct d) from pjt1,pjt2, "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -44354,7 +44354,7 @@ SQL: select distinct a from pjt1 where b>(select avg(distinct d) from pjt1,pjt2, "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -44796,7 +44796,7 @@ SQL: select distinct a from pjt1 where b>(select avg(distinct d) from pjt1,pjt2, "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -44895,7 +44895,7 @@ SQL: select distinct a from pjt1 where b>(select avg(distinct d) from pjt1,pjt2, "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -45592,7 +45592,7 @@ SQL: select max(a) from pjt1,pjt2 where b > (select min(c) from pjt2,pjt3 where "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -45911,7 +45911,7 @@ SQL: select max(a) from pjt1,pjt2 where b > (select min(c) from pjt2,pjt3 where "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -46598,7 +46598,7 @@ SQL: select max(a) from pjt1,pjt2 where b > (select min(c) from pjt2,pjt3 where "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -46997,7 +46997,7 @@ SQL: select max(a) from pjt1,pjt2 where b > (select min(c) from pjt2,pjt3 where "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -47622,7 +47622,7 @@ SQL: select max(a) from pjt1,pjt2 where b > (select min(c) from pjt2,pjt3 where "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -47761,7 +47761,7 @@ SQL: select max(a) from pjt1,pjt2 where b > (select min(c) from pjt2,pjt3 where "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -48404,7 +48404,7 @@ SQL: (select count(distinct a) from pjt1,pjt2) union (select max(c) from pjt2 wh "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -48543,7 +48543,7 @@ SQL: (select count(distinct a) from pjt1,pjt2) union (select max(c) from pjt2 wh "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -49060,7 +49060,7 @@ SQL: (select count(distinct a) from pjt1,pjt2) union (select max(c) from pjt2 wh "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -49489,7 +49489,7 @@ SQL: (select count(distinct a) from pjt1,pjt2) union (select max(c) from pjt2 wh "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -50147,7 +50147,7 @@ SQL: (select count(distinct a) from pjt1,pjt2) union (select max(c) from pjt2 wh "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -50283,7 +50283,7 @@ SQL: (select count(distinct a) from pjt1,pjt2) union (select max(c) from pjt2 wh "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -50467,7 +50467,7 @@ SQL: (select count(distinct a) from pjt1,pjt2) union (select max(c) from pjt2 wh "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -50566,7 +50566,7 @@ SQL: (select count(distinct a) from pjt1,pjt2) union (select max(c) from pjt2 wh "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -51261,7 +51261,7 @@ SQL: (select max(a) from pjt1, pjt2) minus (select distinct e from pjt3,pjt4); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -51400,7 +51400,7 @@ SQL: (select max(a) from pjt1, pjt2) minus (select distinct e from pjt3,pjt4); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -52649,7 +52649,7 @@ SQL: select sum(distinct b) from pjt1, (select distinct c from pjt2, pjt3) where "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -52924,7 +52924,7 @@ SQL: select sum(distinct b) from pjt1, (select distinct c from pjt2, pjt3) where "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -55035,7 +55035,7 @@ SQL: select distinct a from pjt1, pjt2 where b > (select max(e) from pjt3, pjt4) "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -55174,7 +55174,7 @@ SQL: select distinct a from pjt1, pjt2 where b > (select max(e) from pjt3, pjt4) "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -55871,7 +55871,7 @@ SQL: select count(distinct a) from pjt1,pjt2 where b in (select distinct c from "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -56153,7 +56153,7 @@ SQL: select count(distinct a) from pjt1,pjt2 where b in (select distinct c from "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -57637,7 +57637,7 @@ SQL: select count(distinct a) from pjt1,pjt2 where b in (select distinct c from "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -57776,7 +57776,7 @@ SQL: select count(distinct a) from pjt1,pjt2 where b in (select distinct c from "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -61594,7 +61594,7 @@ SQL: select avg(b) from pjt1, pjt2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -61730,7 +61730,7 @@ SQL: select avg(b) from pjt1, pjt2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -61914,7 +61914,7 @@ SQL: select avg(b) from pjt1, pjt2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -62013,7 +62013,7 @@ SQL: select avg(b) from pjt1, pjt2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -62706,7 +62706,7 @@ SQL: select max(a), sum(b) from pjt1, pjt2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -62820,7 +62820,7 @@ SQL: select max(a), sum(b) from pjt1, pjt2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -62959,7 +62959,7 @@ SQL: select max(a), sum(b) from pjt1, pjt2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -63058,7 +63058,7 @@ SQL: select max(a), sum(b) from pjt1, pjt2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -63836,7 +63836,7 @@ SQL: select max(a+c) from pjt1, pjt2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -64060,7 +64060,7 @@ SQL: select max(a+c) from pjt1, pjt2; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -68657,7 +68657,7 @@ SQL: select sum(c1) from test_simp group by c2 ,c3; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -68894,7 +68894,7 @@ SQL: select sum(c1) from test_simp group by c2 ,c3; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -69318,7 +69318,7 @@ SQL: select sum(c1) from test_simp group by c2 ,c3, c4; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -69606,7 +69606,7 @@ SQL: select sum(c1) from test_simp group by c2 ,c3, c4; "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], diff --git a/unittest/sql/rewrite/result/test_transformer_stmt_trans_outline.result b/unittest/sql/rewrite/result/test_transformer_stmt_trans_outline.result index 0c5c2ccb8d..56e1fd4700 100644 --- a/unittest/sql/rewrite/result/test_transformer_stmt_trans_outline.result +++ b/unittest/sql/rewrite/result/test_transformer_stmt_trans_outline.result @@ -6340,7 +6340,7 @@ SQL: select * from t1 where c1 >ANY (select c1 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -6478,7 +6478,7 @@ SQL: select * from t1 where c1 >ANY (select c1 from t2); "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -7684,7 +7684,7 @@ SQL: select * from t1 where c1 >>>>>> de0eba8f08f... GIS sql syntax @@ -13042,7 +13042,7 @@ SQL: select * from t1 where Exists (select c1 from t2 where c2 >ALL (select c1 f "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -13210,7 +13210,7 @@ SQL: select * from t1 where Exists (select c1 from t2 where c2 >ALL (select c1 f "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -13372,7 +13372,7 @@ SQL: select * from t1 where Exists (select c1 from t2 where c2 >ALL (select c1 f "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -16229,7 +16229,7 @@ SQL: select * from t1 where c1 !=ANY (select c1 from t2 where t1.c2=t2.c2 limit "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -16367,7 +16367,7 @@ SQL: select * from t1 where c1 !=ANY (select c1 from t2 where t1.c2=t2.c2 limit "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -125601,7 +125601,7 @@ SQL: select * from (select * from t1 where c1 > 1 + (select avg(c2) from t2)) as "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -125737,7 +125737,7 @@ SQL: select * from (select * from t1 where c1 > 1 + (select avg(c2) from t2)) as "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -125917,7 +125917,7 @@ SQL: select * from (select * from t1 where c1 > 1 + (select avg(c2) from t2)) as "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -126016,7 +126016,7 @@ SQL: select * from (select * from t1 where c1 > 1 + (select avg(c2) from t2)) as "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -127082,7 +127082,7 @@ SQL: select * from t1 where c1 > 1 + (select avg(c2) from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -127218,7 +127218,7 @@ SQL: select * from t1 where c1 > 1 + (select avg(c2) from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -127398,7 +127398,7 @@ SQL: select * from t1 where c1 > 1 + (select avg(c2) from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -127497,7 +127497,7 @@ SQL: select * from t1 where c1 > 1 + (select avg(c2) from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -128978,7 +128978,7 @@ SQL: select * from (select * from t1 where c1 > 1 + (select avg(c2) from t2)) as "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -129114,7 +129114,7 @@ SQL: select * from (select * from t1 where c1 > 1 + (select avg(c2) from t2)) as "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -129294,7 +129294,7 @@ SQL: select * from (select * from t1 where c1 > 1 + (select avg(c2) from t2)) as "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -129393,7 +129393,7 @@ SQL: select * from (select * from t1 where c1 > 1 + (select avg(c2) from t2)) as "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -130874,7 +130874,7 @@ SQL: select * from t1, t3 where t1.c1 > 1 + (select avg(c2) from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -131010,7 +131010,7 @@ SQL: select * from t1, t3 where t1.c1 > 1 + (select avg(c2) from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -131190,7 +131190,7 @@ SQL: select * from t1, t3 where t1.c1 > 1 + (select avg(c2) from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -131289,7 +131289,7 @@ SQL: select * from t1, t3 where t1.c1 > 1 + (select avg(c2) from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -132906,7 +132906,7 @@ SQL: select * from (select * from t1 where c1 > 1 + (select avg(c2) from t2)) as "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -133042,7 +133042,7 @@ SQL: select * from (select * from t1 where c1 > 1 + (select avg(c2) from t2)) as "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -133222,7 +133222,7 @@ SQL: select * from (select * from t1 where c1 > 1 + (select avg(c2) from t2)) as "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -133321,7 +133321,7 @@ SQL: select * from (select * from t1 where c1 > 1 + (select avg(c2) from t2)) as "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -134819,7 +134819,7 @@ SQL: select * from t1, t3 where t1.c1 > 1 + (select avg(c2) from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -134955,7 +134955,7 @@ SQL: select * from t1, t3 where t1.c1 > 1 + (select avg(c2) from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -135135,7 +135135,7 @@ SQL: select * from t1, t3 where t1.c1 > 1 + (select avg(c2) from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -135234,7 +135234,7 @@ SQL: select * from t1, t3 where t1.c1 > 1 + (select avg(c2) from t2); "dl":"", "type":0 }, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -144018,7 +144018,7 @@ SQL: select * from t1 where c1 >ANY (select c1 from t2) and t1.c1 - 10 < 5; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -144195,7 +144195,7 @@ SQL: select * from t1 where c1 >ANY (select c1 from t2) and t1.c1 - 10 < 5; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -145568,7 +145568,7 @@ SQL: select * from (select * from t1 where c1 >ANY (select c1 from t2)) as v whe "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -145692,7 +145692,7 @@ SQL: select * from (select * from t1 where c1 >ANY (select c1 from t2)) as v whe "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -147363,7 +147363,7 @@ SQL: select * from t1 where c1 >ANY (select c1 from t2) and t1.c1 < 10; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -148978,7 +148978,7 @@ SQL: select * from (select * from t1 where c1 10; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -152892,7 +152892,7 @@ SQL: select * from t1 where c1 10; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -156162,7 +156162,7 @@ SQL: select * from t1 where 1 < ANY(select c1 from t2) and t1.c1 - 10 = 10; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null >>>>>>> de0eba8f08f... GIS sql syntax }, @@ -156472,7 +156472,7 @@ SQL: select * from t1 where 1 < ANY(select c1 from t2) and t1.c1 - 10 = 10; "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } >>>>>>> de0eba8f08f... GIS sql syntax @@ -157966,7 +157966,7 @@ SQL: select * from (select * from t1 where 1 < ANY(select c1 from t2 limit 1)) a "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } >>>>>>> de0eba8f08f... GIS sql syntax @@ -163121,7 +163121,7 @@ SQL: select * from (select * from t1 where Exists (select c1 from t2 where t1.c2 "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -163283,7 +163283,7 @@ SQL: select * from (select * from t1 where Exists (select c1 from t2 where t1.c2 "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -165176,7 +165176,7 @@ SQL: select * from t1 where Exists (select c1 from t2 where t1.c2=t2.c2) and t1. "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, "is_alias":false, @@ -165344,7 +165344,7 @@ SQL: select * from t1 where Exists (select c1 from t2 where t1.c2=t2.c2) and t1. "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null }, { @@ -165506,7 +165506,7 @@ SQL: select * from t1 where Exists (select c1 from t2 where t1.c2=t2.c2) and t1. "type":0 }, "linear_inter_expr":null, - "is_nested_aggr":false, + "expr_in_inner_stmt":false, "pl_agg_udf_expr":null } ], @@ -168260,7 +168260,7 @@ SQL: select * from (select (c1 + 1) as k from t1 where c1 init(backup_data_type, batch_count); + ObLSBackupCtx ls_backup_ctx; + ret = task_mgr->init(backup_data_type, batch_count, ls_backup_ctx); EXPECT_EQ(OB_SUCCESS, ret); ret = provider->init(total_item_count); EXPECT_EQ(OB_SUCCESS, ret); diff --git a/unittest/storage/blocksstable/test_sstable_meta.cpp b/unittest/storage/blocksstable/test_sstable_meta.cpp index df6f3bd8a7..43cf59a464 100644 --- a/unittest/storage/blocksstable/test_sstable_meta.cpp +++ b/unittest/storage/blocksstable/test_sstable_meta.cpp @@ -604,6 +604,70 @@ TEST_F(TestSSTableMeta, test_sstable_deep_copy) ASSERT_EQ(full_sstable.meta_->is_inited_, tiny_sstable->meta_->is_inited_); } +TEST_F(TestSSTableMeta, test_sstable_meta_deep_copy) +{ + int ret = OB_SUCCESS; + ObSSTableMeta src_meta; + // add salt + src_meta.basic_meta_.data_checksum_ = 20240514; + + src_meta.column_checksum_count_ = 3; + src_meta.column_checksums_ = (int64_t*)ob_malloc_align(4<<10, 3 * sizeof(int64_t), ObMemAttr()); + src_meta.column_checksums_[0] = 1111; + src_meta.column_checksums_[1] = 2222; + src_meta.column_checksums_[2] = 3333; + + src_meta.tx_ctx_.tx_descs_ = (ObTxContext::ObTxDesc*)ob_malloc_align(4<<10, 2 * sizeof(ObTxContext::ObTxDesc), ObMemAttr()); + ret = src_meta.tx_ctx_.push_back({987, 654}); + ASSERT_EQ(OB_SUCCESS, ret); + ret = src_meta.tx_ctx_.push_back({123, 456}); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(2, src_meta.tx_ctx_.count_); + ASSERT_EQ(2 * sizeof(ObTxContext::ObTxDesc), src_meta.tx_ctx_.get_variable_size()); + src_meta.tx_ctx_.len_ = src_meta.tx_ctx_.get_serialize_size(); + + // test deep copy from dynamic memory meta to flat memory meta + const int64_t buf_size = 8 << 10; //8K + int64_t pos = 0; + char *flat_buf_1 = (char*)ob_malloc(buf_size, ObMemAttr()); + int64_t deep_copy_size = src_meta.get_deep_copy_size(); + ObSSTableMeta *flat_meta_1; + ret = src_meta.deep_copy(flat_buf_1, deep_copy_size, pos, flat_meta_1); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(deep_copy_size, pos); + OB_LOG(INFO, "cooper", K(src_meta), K(sizeof(ObSSTableMeta)), K(deep_copy_size)); + OB_LOG(INFO, "cooper", K(*flat_meta_1)); + // can't use MEMCMP between dynamic memory and flat memory, because one is stack, the other is heap + ASSERT_EQ(src_meta.basic_meta_, flat_meta_1->basic_meta_); + // ASSERT_EQ(0, MEMCMP((char*)&src_meta.data_root_info_, (char*)&flat_meta_1->data_root_info_, sizeof(src_meta.data_root_info_))); + // ASSERT_EQ(0, MEMCMP((char*)&src_meta.macro_info_, (char*)&dst_meta->macro_info_, sizeof(src_meta.macro_info_))); + // ASSERT_EQ(0, MEMCMP((char*)&src_meta.cg_sstables_, (char*)&dst_meta->cg_sstables_, sizeof(src_meta.cg_sstables_))); + ASSERT_EQ(0, MEMCMP(src_meta.column_checksums_, flat_meta_1->column_checksums_, src_meta.column_checksum_count_ * sizeof(int64_t))); + ASSERT_EQ(src_meta.tx_ctx_.len_, flat_meta_1->tx_ctx_.len_); + ASSERT_EQ(src_meta.tx_ctx_.count_, flat_meta_1->tx_ctx_.count_); + ASSERT_EQ(0, MEMCMP(src_meta.tx_ctx_.tx_descs_, flat_meta_1->tx_ctx_.tx_descs_, flat_meta_1->tx_ctx_.get_variable_size())); + + // test deep copy from flat memory meta to flat memory meta + pos = 0; + char *flat_buf_2 = (char*)ob_malloc_align(4<<10, buf_size, ObMemAttr()); + deep_copy_size = flat_meta_1->get_deep_copy_size(); + ObSSTableMeta *flat_meta_2; + ret = flat_meta_1->deep_copy(flat_buf_2, deep_copy_size, pos, flat_meta_2); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(deep_copy_size, pos); + OB_LOG(INFO, "cooper", K(*flat_meta_1)); + OB_LOG(INFO, "cooper", K(*flat_meta_2)); + ASSERT_EQ(0, MEMCMP((char*)&flat_meta_1->basic_meta_, (char*)&flat_meta_2->basic_meta_, sizeof(flat_meta_1->basic_meta_))); + ASSERT_EQ(0, MEMCMP(&flat_meta_1->data_root_info_, &flat_meta_2->data_root_info_, sizeof(flat_meta_1->data_root_info_))); + ASSERT_EQ(0, MEMCMP(&flat_meta_1->macro_info_, &flat_meta_2->macro_info_, sizeof(flat_meta_1->macro_info_))); + ASSERT_EQ(0, MEMCMP((char*)&flat_meta_1->cg_sstables_, (char*)&flat_meta_2->cg_sstables_, sizeof(flat_meta_1->cg_sstables_))); + ASSERT_EQ(0, MEMCMP(flat_meta_1->column_checksums_, flat_meta_2->column_checksums_, flat_meta_1->column_checksum_count_ * sizeof(int64_t))); + ASSERT_EQ(flat_meta_2->tx_ctx_.len_, flat_meta_1->tx_ctx_.len_); + ASSERT_EQ(flat_meta_2->tx_ctx_.count_, flat_meta_1->tx_ctx_.count_); + ASSERT_NE(flat_meta_1->tx_ctx_.tx_descs_, flat_meta_2->tx_ctx_.tx_descs_); + ASSERT_EQ(0, MEMCMP(flat_meta_2->tx_ctx_.tx_descs_, flat_meta_1->tx_ctx_.tx_descs_, flat_meta_1->tx_ctx_.get_variable_size())); +} + TEST_F(TestMigrationSSTableParam, test_empty_sstable_serialize_and_deserialize) { ObMigrationSSTableParam mig_param; @@ -698,7 +762,6 @@ TEST_F(TestMigrationSSTableParam, test_migrate_sstable) ASSERT_TRUE(dest_sstable_param.encrypt_id_ == src_sstable_param.encrypt_id_); } - } // end namespace unittest } // end namespace oceanbase diff --git a/unittest/storage/tx/it/test_register_mds.cpp b/unittest/storage/tx/it/test_register_mds.cpp index 51956f9865..4ad59a8f28 100644 --- a/unittest/storage/tx/it/test_register_mds.cpp +++ b/unittest/storage/tx/it/test_register_mds.cpp @@ -142,6 +142,7 @@ public: auto test_name = test_info->name(); MTL_MEM_ALLOC_MGR.init(); _TRANS_LOG(INFO, ">>>> starting test : %s", test_name); + LOG_INFO(">>>>>>starting>>>>>>>>", K(test_name)); } virtual void TearDown() override { @@ -151,6 +152,7 @@ public: _TRANS_LOG(INFO, ">>>> tearDown test : %s", test_name); ObClockGenerator::destroy(); ObMallocAllocator::get_instance()->recycle_tenant_allocator(1001); + LOG_INFO(">>>>>teardown>>>>>>>>", K(test_name)); } MsgBus bus_; }; diff --git a/unittest/storage/tx/it/test_tx.cpp b/unittest/storage/tx/it/test_tx.cpp index f2a6a2c677..ae180bff7e 100644 --- a/unittest/storage/tx/it/test_tx.cpp +++ b/unittest/storage/tx/it/test_tx.cpp @@ -340,29 +340,6 @@ TEST_F(ObTestTx, rollback_savepoint_with_need_retry_error) } } -TEST_F(ObTestTx, create_savepoint_with_sanity_check_tx_abort) -{ - START_ONE_TX_NODE(n1); - PREPARE_TX(n1, tx); - PREPARE_TX_PARAM(tx_param); - GET_READ_SNAPSHOT(n1, tx, tx_param, snapshot); - tx.flags_.PART_ABORTED_ = 1; - ObTxSEQ sp; - ASSERT_EQ(OB_TRANS_NEED_ROLLBACK, n1->create_implicit_savepoint(tx, tx_param, sp)); - -} - -TEST_F(ObTestTx, rollback_savepoint_with_sanity_check_tx_abort) -{ - START_ONE_TX_NODE(n1); - PREPARE_TX(n1, tx); - PREPARE_TX_PARAM(tx_param); - GET_READ_SNAPSHOT(n1, tx, tx_param, snapshot); - CREATE_IMPLICIT_SAVEPOINT(n1, tx, tx_param, sp); - tx.flags_.PART_ABORTED_ = 1; - ASSERT_EQ(OB_TRANS_NEED_ROLLBACK, n1->rollback_to_implicit_savepoint(tx, sp, n1->ts_after_ms(5), nullptr)); -} - TEST_F(ObTestTx, switch_to_follower_gracefully) { int ret = OB_SUCCESS; diff --git a/unittest/storage/tx/mock_utils/basic_fake_define.h b/unittest/storage/tx/mock_utils/basic_fake_define.h index 2ec02762fe..d821326ecb 100644 --- a/unittest/storage/tx/mock_utils/basic_fake_define.h +++ b/unittest/storage/tx/mock_utils/basic_fake_define.h @@ -37,7 +37,10 @@ namespace transaction { class ObFakeTxDataTable : public ObTxDataTable { public: ObSliceAlloc slice_allocator_; - ObTenantTxDataAllocator *FAKE_ALLOCATOR = (ObTenantTxDataAllocator *)0x1; + ObTenantTxDataAllocator __FAKE_ALLOCATOR_OBJ; + ObTenantTxDataAllocator *FAKE_ALLOCATOR = &__FAKE_ALLOCATOR_OBJ; + ObTenantTxDataOpAllocator __FAKE_ALLOCATOR_OBJ2; + ObTenantTxDataOpAllocator *FAKE_ALLOCATOR2 = &__FAKE_ALLOCATOR_OBJ2; public: ObFakeTxDataTable() : arena_allocator_(), map_(arena_allocator_, 1 << 20 /*2097152*/) @@ -51,6 +54,8 @@ public: OB_ASSERT(OB_SUCCESS == slice_allocator_.init( sizeof(ObTxData), OB_MALLOC_NORMAL_BLOCK_SIZE, common::default_blk_alloc, mem_attr)); slice_allocator_.set_nway(32); + FAKE_ALLOCATOR->init("FAKE_A"); + FAKE_ALLOCATOR2->init(); is_inited_ = true; } virtual int init(ObLS *ls, ObTxCtxTable *tx_ctx_table) override @@ -70,6 +75,7 @@ public: ObTxData *tx_data = new (ptr) ObTxData(); tx_data->ref_cnt_ = 100; tx_data->tx_data_allocator_ = FAKE_ALLOCATOR; + tx_data->op_allocator_ = FAKE_ALLOCATOR2; tx_data_guard.init(tx_data); return OB_ISNULL(tx_data) ? OB_ALLOCATE_MEMORY_FAILED : OB_SUCCESS; } @@ -83,7 +89,6 @@ public: to->tx_data_allocator_ = FAKE_ALLOCATOR; to_guard.init(to); OX (*to = *from); - OZ (deep_copy_undo_status_list_(from->undo_status_list_, to->undo_status_list_)); return ret; } virtual void free_tx_data(ObTxData *tx_data) diff --git a/unittest/storage/tx_table/test_tx_ctx_table.cpp b/unittest/storage/tx_table/test_tx_ctx_table.cpp index 67ca70d61a..0e2d175fd2 100644 --- a/unittest/storage/tx_table/test_tx_ctx_table.cpp +++ b/unittest/storage/tx_table/test_tx_ctx_table.cpp @@ -190,6 +190,7 @@ public: ObIMemtableMgr *mt_mgr_; ObTxCtxMemtableMgr *ctx_mt_mgr_; ObTenantTxDataAllocator tx_data_allocator_; + ObTenantTxDataOpAllocator tx_data_op_allocator_; ObTenantBase tenant_base_; }; @@ -304,6 +305,7 @@ TEST_F(TestTxCtxTable, test_tx_ctx_memtable_mgr) attr.tenant_id_ = MTL_ID(); tx_data_allocator_.init("test"); tx_data_table.tx_data_allocator_ = &tx_data_allocator_; + tx_data_op_allocator_.init(); ObTxPalfParam palf_param((logservice::ObLogHandler *)(0x01), (transaction::ObDupTableLSHandler *)(0x02)); @@ -439,9 +441,12 @@ int ObLSTxCtxMgr::init(const int64_t tenant_id, //} if (OB_FAIL(ls_tx_ctx_map_.init(lib::ObMemAttr(tenant_id, "LSTxCtxMgr")))) { TRANS_LOG(WARN, "ls_tx_ctx_map_ init fail", KR(ret)); + } else if (OB_FAIL(tx_ls_state_mgr_.init(ls_id))) { + TRANS_LOG(WARN, "init tx_ls_state_mgr_ failed", KR(ret)); + } else if (OB_FAIL(tx_ls_state_mgr_.switch_tx_ls_state(ObTxLSStateMgr::TxLSAction::START))) { + TRANS_LOG(WARN, "start ls_tx_ctx_mgr failed",K(ret),K(tx_ls_state_mgr_)); } else { is_inited_ = true; - state_ = State::F_WORKING; tenant_id_ = tenant_id; ls_id_ = ls_id; tx_table_ = tx_table;